#23237

Committed 23 Sep 2024 03:40PM UTC coverage: 20.657%. First build

Build # #23237

Build Type

Pull #10781

github

Committed by

pdurbin

Commit Message

Merge branch 'develop' into 10623-globus-improvements #10623

Pull Request Pull Request #10781: Improved handling of Globus uploads

Run Details

4 of 417 new or added lines in 15 files covered. (0.96%)

17604 of 85219 relevant lines covered (20.66%)

0.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java

package edu.harvard.iq.dataverse.globus;

import edu.harvard.iq.dataverse.settings.JvmSettings;
import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
import edu.harvard.iq.dataverse.util.SystemConfig;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.Resource;
import jakarta.ejb.EJB;
import jakarta.ejb.Singleton;
import jakarta.ejb.Startup;
import jakarta.enterprise.concurrent.ManagedScheduledExecutorService;
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.logging.FileHandler;
import java.util.logging.Logger;

/**
 * 
 * This Singleton monitors ongoing Globus tasks by checking with the centralized
 * Globus API on the status of all the registered ongoing tasks. 
 * When a successful completion of a task is detected, the service triggers
 * the execution of the associated tasks (for example, finalizing adding datafiles
 * to the dataset on completion of a remote Globus upload). When a task fails or 
 * terminates abnormally, a message is logged and the task record is deleted 
 * from the database. 
 * 
 * @author landreev
 */
@Singleton
@Startup
public class TaskMonitoringServiceBean {
    private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.globus.TaskMonitoringServiceBean");
    
    @Resource
    ManagedScheduledExecutorService scheduler;
    
    @EJB
    SystemConfig systemConfig;
    @EJB
    SettingsServiceBean settingsSvc;
    @EJB 
    GlobusServiceBean globusService;
    
    private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss");
    
    @PostConstruct
    public void init() {
        if (JvmSettings.GLOBUS_TASK_MONITORING_SERVER.lookupOptional(Boolean.class).orElse(false)) {
            logger.info("Starting Globus task monitoring service");
            int pollingInterval = SystemConfig.getIntLimitFromStringOrDefault(
                settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusPollingInterval), 600);
            this.scheduler.scheduleWithFixedDelay(this::checkOngoingTasks,
                    0, pollingInterval,
                    TimeUnit.SECONDS);
        } else {
            logger.info("Skipping Globus task monitor initialization");
        }
    }
    
    /**
     * This method will be executed on a timer-like schedule, continuously 
     * monitoring all the ongoing external Globus tasks (transfers). 
     */
    public void checkOngoingTasks() {
        logger.fine("Performing a scheduled external Globus task check");
        List<GlobusTaskInProgress> tasks = globusService.findAllOngoingTasks();

        tasks.forEach(t -> {
            FileHandler taskLogHandler = getTaskLogHandler(t);
            Logger taskLogger = getTaskLogger(t, taskLogHandler);
            
            GlobusTaskState retrieved = globusService.getTask(t.getGlobusToken(), t.getTaskId(), taskLogger);
            if (GlobusUtil.isTaskCompleted(retrieved)) {
                // Do our thing, finalize adding the files to the dataset
                globusService.processCompletedTask(t, GlobusUtil.isTaskSucceeded(retrieved), GlobusUtil.getTaskStatus(retrieved), taskLogger);
                // Whether it finished successfully, or failed in the process, 
                // there's no need to keep monitoring this task, so we can 
                // delete it.
                //globusService.deleteExternalUploadRecords(t.getTaskId());
                globusService.deleteTask(t);
            }
            
            if (taskLogHandler != null) {
                // @todo it should be prudent to cache these loggers and handlers 
                // between monitoring runs (should be fairly easy to do)
                taskLogHandler.close();
            }
        });
    }
    
    private FileHandler getTaskLogHandler(GlobusTaskInProgress task) {
        if (task == null) {
            return null; 
        }
        
        Date startDate = new Date(task.getStartTime().getTime());
        String logTimeStamp = logFormatter.format(startDate);
        
        String logFileName = System.getProperty("com.sun.aas.instanceRoot") + File.separator + "logs" + File.separator + "globusUpload_" + task.getDataset().getId() + "_" + logTimeStamp
                + ".log";
        FileHandler fileHandler;
        try {
            fileHandler = new FileHandler(logFileName);
        } catch (IOException | SecurityException ex) {
            // @todo log this error somehow?
            fileHandler = null;
        }
        return fileHandler;
    }
    
    private Logger getTaskLogger(GlobusTaskInProgress task, FileHandler logFileHandler) {
        if (logFileHandler == null) {
            return null;
        }
        Date startDate = new Date(task.getStartTime().getTime());
        String logTimeStamp = logFormatter.format(startDate);
        
        Logger taskLogger = Logger.getLogger(
                "edu.harvard.iq.dataverse.upload.client.DatasetServiceBean." + "GlobusUpload" + logTimeStamp);
            taskLogger.setUseParentHandlers(false);
       
        taskLogger.addHandler(logFileHandler);
        
        return taskLogger;        
    }
    
}

1	package edu.harvard.iq.dataverse.globus;
2
3	import edu.harvard.iq.dataverse.settings.JvmSettings;
4	import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
5	import edu.harvard.iq.dataverse.util.SystemConfig;
6	import jakarta.annotation.PostConstruct;
7	import jakarta.annotation.Resource;
8	import jakarta.ejb.EJB;
9	import jakarta.ejb.Singleton;
10	import jakarta.ejb.Startup;
11	import jakarta.enterprise.concurrent.ManagedScheduledExecutorService;
12	import java.io.File;
13	import java.io.IOException;
14	import java.text.SimpleDateFormat;
15	import java.util.Date;
16	import java.util.List;
17	import java.util.concurrent.TimeUnit;
18	import java.util.logging.FileHandler;
19	import java.util.logging.Logger;
20
21	/**
22	*
23	* This Singleton monitors ongoing Globus tasks by checking with the centralized
24	* Globus API on the status of all the registered ongoing tasks.
25	* When a successful completion of a task is detected, the service triggers
26	* the execution of the associated tasks (for example, finalizing adding datafiles
27	* to the dataset on completion of a remote Globus upload). When a task fails or
28	* terminates abnormally, a message is logged and the task record is deleted
29	* from the database.
30	*
31	* @author landreev
32	*/
33	@Singleton
34	@Startup
NEW 35	public class TaskMonitoringServiceBean {	×
NEW 36	private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.globus.TaskMonitoringServiceBean");	×
37
38	@Resource
39	ManagedScheduledExecutorService scheduler;
40
41	@EJB
42	SystemConfig systemConfig;
43	@EJB
44	SettingsServiceBean settingsSvc;
45	@EJB
46	GlobusServiceBean globusService;
47
NEW 48	private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss");	×
49
50	@PostConstruct
51	public void init() {
NEW 52	if (JvmSettings.GLOBUS_TASK_MONITORING_SERVER.lookupOptional(Boolean.class).orElse(false)) {	×
NEW 53	logger.info("Starting Globus task monitoring service");	×
NEW 54	int pollingInterval = SystemConfig.getIntLimitFromStringOrDefault(	×
NEW 55	settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusPollingInterval), 600);	×
NEW 56	this.scheduler.scheduleWithFixedDelay(this::checkOngoingTasks,	×
57	0, pollingInterval,
58	TimeUnit.SECONDS);
NEW 59	} else {	×
NEW 60	logger.info("Skipping Globus task monitor initialization");	×
61	}
NEW 62	}	×
63
64	/**
65	* This method will be executed on a timer-like schedule, continuously
66	* monitoring all the ongoing external Globus tasks (transfers).
67	*/
68	public void checkOngoingTasks() {
NEW 69	logger.fine("Performing a scheduled external Globus task check");	×
NEW 70	List<GlobusTaskInProgress> tasks = globusService.findAllOngoingTasks();	×
71
NEW 72	tasks.forEach(t -> {	×
NEW 73	FileHandler taskLogHandler = getTaskLogHandler(t);	×
NEW 74	Logger taskLogger = getTaskLogger(t, taskLogHandler);	×
75
NEW 76	GlobusTaskState retrieved = globusService.getTask(t.getGlobusToken(), t.getTaskId(), taskLogger);	×
NEW 77	if (GlobusUtil.isTaskCompleted(retrieved)) {	×
78	// Do our thing, finalize adding the files to the dataset
NEW 79	globusService.processCompletedTask(t, GlobusUtil.isTaskSucceeded(retrieved), GlobusUtil.getTaskStatus(retrieved), taskLogger);	×
80	// Whether it finished successfully, or failed in the process,
81	// there's no need to keep monitoring this task, so we can
82	// delete it.
83	//globusService.deleteExternalUploadRecords(t.getTaskId());
NEW 84	globusService.deleteTask(t);	×
85	}
86
NEW 87	if (taskLogHandler != null) {	×
88	// @todo it should be prudent to cache these loggers and handlers
89	// between monitoring runs (should be fairly easy to do)
NEW 90	taskLogHandler.close();	×
91	}
NEW 92	});	×
NEW 93	}	×
94
95	private FileHandler getTaskLogHandler(GlobusTaskInProgress task) {
NEW 96	if (task == null) {	×
NEW 97	return null;	×
98	}
99
NEW 100	Date startDate = new Date(task.getStartTime().getTime());	×
NEW 101	String logTimeStamp = logFormatter.format(startDate);	×
102
NEW 103	String logFileName = System.getProperty("com.sun.aas.instanceRoot") + File.separator + "logs" + File.separator + "globusUpload_" + task.getDataset().getId() + "_" + logTimeStamp	×
104	+ ".log";
105	FileHandler fileHandler;
106	try {
NEW 107	fileHandler = new FileHandler(logFileName);	×
NEW 108	} catch (IOException \| SecurityException ex) {	×
109	// @todo log this error somehow?
NEW 110	fileHandler = null;	×
NEW 111	}	×
NEW 112	return fileHandler;	×
113	}
114
115	private Logger getTaskLogger(GlobusTaskInProgress task, FileHandler logFileHandler) {
NEW 116	if (logFileHandler == null) {	×
NEW 117	return null;	×
118	}
NEW 119	Date startDate = new Date(task.getStartTime().getTime());	×
NEW 120	String logTimeStamp = logFormatter.format(startDate);	×
121
NEW 122	Logger taskLogger = Logger.getLogger(	×
123	"edu.harvard.iq.dataverse.upload.client.DatasetServiceBean." + "GlobusUpload" + logTimeStamp);
NEW 124	taskLogger.setUseParentHandlers(false);	×
125
NEW 126	taskLogger.addHandler(logFileHandler);	×
127
NEW 128	return taskLogger;	×
129	}
130
131	}

IQSS / dataverse / #23237

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous