#22987

Committed 23 Aug 2024 06:44PM UTC coverage: 20.61% (-0.2%) from 20.791%

Build # #22987

Build Type

Pull #10781

github

Committed by

landreev

Commit Message

added an upfront locks check to the /addGlobusFiles api #10623

Pull Request Pull Request #10781: Improved handling of Globus uploads

Run Details

4 of 417 new or added lines in 15 files covered. (0.96%)

4194 existing lines in 35 files now uncovered.

17388 of 84365 relevant lines covered (20.61%)

0.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

34.88

/src/main/java/edu/harvard/iq/dataverse/globus/GlobusUtil.java

package edu.harvard.iq.dataverse.globus;

import java.util.List;

import edu.harvard.iq.dataverse.DataFile;
import edu.harvard.iq.dataverse.Dataset;
import edu.harvard.iq.dataverse.dataaccess.DataAccess;
import edu.harvard.iq.dataverse.dataaccess.GlobusAccessibleStore;
import jakarta.json.Json;
import jakarta.json.JsonObject;
import jakarta.json.JsonObjectBuilder;

public class GlobusUtil {

    public static JsonObject getFilesMap(List<DataFile> dataFiles, Dataset d) {
        JsonObjectBuilder filesBuilder = Json.createObjectBuilder();
        for (DataFile df : dataFiles) {
            String storageId = df.getStorageIdentifier();
            String[] parts = DataAccess
                    .getDriverIdAndStorageLocation(DataAccess.getLocationFromStorageId(storageId, d));
            String driverId = parts[0];
            String fileLocation = parts[1];
            if (GlobusAccessibleStore.isDataverseManaged(driverId)) {
                String endpointWithBasePath = GlobusAccessibleStore.getTransferEnpointWithPath(driverId);
                fileLocation = endpointWithBasePath + "/" + fileLocation;
            } else {
                fileLocation = storageId.substring(storageId.lastIndexOf("//") + 2);
            }
            filesBuilder.add(df.getId().toString(), fileLocation);
        }
        return filesBuilder.build();
    }
    
    public static boolean isTaskCompleted(GlobusTaskState task) {
        if (task != null) {
            String status = task.getStatus();
            if (status != null) {
                if (status.equalsIgnoreCase("ACTIVE")) {
                    if (task.getNice_status().equalsIgnoreCase("ok")
                            || task.getNice_status().equalsIgnoreCase("queued")) {
                        return false;
                    }
                }
            }
        }
        return true;
    }
    
    public static boolean isTaskSucceeded(GlobusTaskState task) {
        String status = null;
        if (task != null) {
            status = task.getStatus();
            if (status != null) {
                status = status.toUpperCase();
                if (status.equals("ACTIVE") || status.startsWith("FAILED") || status.startsWith("INACTIVE")) {
                    // There are cases where a failed task may still be showing 
                    // as "ACTIVE". But it is definitely safe to assume that it 
                    // has not completed *successfully*.
                    return false;
                } 
                return true;
            } 
        } 
        return false;
    }
    /**
     * Produces a human-readable Status label of a completed task
     * @param GlobusTaskState task - a looked-up state of a task as reported by Globus API
     */
    public static String getTaskStatus(GlobusTaskState task) {
        String status = null;
        if (task != null) {
            status = task.getStatus();
            if (status != null) {
                // The task is in progress but is not ok or queued
                // (L.A.) I think the assumption here is that this method is called 
                // exclusively on tasks that have already completed. So that's why
                // it is safe to assume that "ACTIVE" means "FAILED". 
                if (status.equalsIgnoreCase("ACTIVE")) {
                    status = "FAILED" + "#" + task.getNice_status() + "#" + task.getNice_status_short_description();
                } else {
                    // The task is either succeeded, failed or inactive.
                    status = status + "#" + task.getNice_status() + "#" + task.getNice_status_short_description();
                }
            } else {
                status = "FAILED";
            }
        } else {
            status = "FAILED";
        }
        return status;
    }
}

1	package edu.harvard.iq.dataverse.globus;
2
3	import java.util.List;
4
5	import edu.harvard.iq.dataverse.DataFile;
6	import edu.harvard.iq.dataverse.Dataset;
7	import edu.harvard.iq.dataverse.dataaccess.DataAccess;
8	import edu.harvard.iq.dataverse.dataaccess.GlobusAccessibleStore;
9	import jakarta.json.Json;
10	import jakarta.json.JsonObject;
11	import jakarta.json.JsonObjectBuilder;
12
13	public class GlobusUtil {	×
14
15	public static JsonObject getFilesMap(List<DataFile> dataFiles, Dataset d) {
16	JsonObjectBuilder filesBuilder = Json.createObjectBuilder();	1✔
17	for (DataFile df : dataFiles) {	1✔
18	String storageId = df.getStorageIdentifier();	1✔
19	String[] parts = DataAccess	1✔
20	.getDriverIdAndStorageLocation(DataAccess.getLocationFromStorageId(storageId, d));	1✔
21	String driverId = parts[0];	1✔
22	String fileLocation = parts[1];	1✔
23	if (GlobusAccessibleStore.isDataverseManaged(driverId)) {	1✔
24	String endpointWithBasePath = GlobusAccessibleStore.getTransferEnpointWithPath(driverId);	1✔
25	fileLocation = endpointWithBasePath + "/" + fileLocation;	1✔
26	} else {	1✔
27	fileLocation = storageId.substring(storageId.lastIndexOf("//") + 2);	1✔
28	}
29	filesBuilder.add(df.getId().toString(), fileLocation);	1✔
30	}	1✔
31	return filesBuilder.build();	1✔
32	}
33
34	public static boolean isTaskCompleted(GlobusTaskState task) {
NEW 35	if (task != null) {	×
NEW 36	String status = task.getStatus();	×
NEW 37	if (status != null) {	×
NEW 38	if (status.equalsIgnoreCase("ACTIVE")) {	×
NEW 39	if (task.getNice_status().equalsIgnoreCase("ok")	×
NEW 40	\|\| task.getNice_status().equalsIgnoreCase("queued")) {	×
NEW 41	return false;	×
42	}
43	}
44	}
45	}
NEW 46	return true;	×
47	}
48
49	public static boolean isTaskSucceeded(GlobusTaskState task) {
NEW 50	String status = null;	×
NEW 51	if (task != null) {	×
NEW 52	status = task.getStatus();	×
NEW 53	if (status != null) {	×
NEW 54	status = status.toUpperCase();	×
NEW 55	if (status.equals("ACTIVE") \|\| status.startsWith("FAILED") \|\| status.startsWith("INACTIVE")) {	×
56	// There are cases where a failed task may still be showing
57	// as "ACTIVE". But it is definitely safe to assume that it
58	// has not completed successfully.
NEW 59	return false;	×
60	}
NEW 61	return true;	×
62	}
63	}
NEW 64	return false;	×
65	}
66	/**
67	* Produces a human-readable Status label of a completed task
68	* @param GlobusTaskState task - a looked-up state of a task as reported by Globus API
69	*/
70	public static String getTaskStatus(GlobusTaskState task) {
NEW 71	String status = null;	×
NEW 72	if (task != null) {	×
NEW 73	status = task.getStatus();	×
NEW 74	if (status != null) {	×
75	// The task is in progress but is not ok or queued
76	// (L.A.) I think the assumption here is that this method is called
77	// exclusively on tasks that have already completed. So that's why
78	// it is safe to assume that "ACTIVE" means "FAILED".
NEW 79	if (status.equalsIgnoreCase("ACTIVE")) {	×
NEW 80	status = "FAILED" + "#" + task.getNice_status() + "#" + task.getNice_status_short_description();	×
81	} else {
82	// The task is either succeeded, failed or inactive.
NEW 83	status = status + "#" + task.getNice_status() + "#" + task.getNice_status_short_description();	×
84	}
85	} else {
NEW 86	status = "FAILED";	×
87	}
88	} else {
NEW 89	status = "FAILED";	×
90	}
NEW 91	return status;	×
92	}
93	}

IQSS / dataverse / #22987

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous