30672691247

Committed 31 Jul 2026 11:22PM UTC coverage: 56.743% (+0.07%) from 56.671%

Build # 30672691247

Build Type

Pull #2173

github

Committed by

web-flow

Commit Message

Merge fd95a7456 into 746816931

Pull Request Pull Request #2173: Fix GIL shutdown hang.

Coverage Stats

46 of 55 new or added lines in 3 files covered. (83.64%)

14869 of 26204 relevant lines covered (56.74%)

463203.35 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.22

/trick_source/sim_services/InputProcessor/IPPython.cpp

/*
   PURPOSE: ( Python input processor )
   REFERENCE: ( Trick Simulation Environment )
   ASSUMPTIONS AND LIMITATIONS: ( None )
   CLASS: ( N/A )
   LIBRARY DEPENDENCY: ( None )
   PROGRAMMERS: ( Alex Lin NASA 2009 )
*/

#include <Python.h>

#include "trick/IPPython.hh"

#include "trick/MemoryManager.hh"
#include "trick/exec_proto.h"
#include "trick/io_alloc.h"
#include "trick/message_proto.h"
#include "trick/message_type.h"

#include <atomic>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <pthread.h>
#include <sstream>
#include <string>
#include <unistd.h>

Trick::IPPython * the_pip ;

//Constructor
Trick::IPPython::IPPython() { the_pip = this; }

// Need to save the state of the main thread to allow child threads to run PyRun variants.
static PyThreadState *_save = NULL;

// Number of threads currently inside parse()/parse_condition(), i.e. threads that hold
// the Python GIL or are waiting to acquire it. See GILGuard and shutdown() below.
static std::atomic<int> active_parse_count(0);

namespace
{
    /**
     * Holds the Python GIL for a scope, with three guarantees that the bare
     * PyGILState_Ensure()/PyGILState_Release() pair does not provide.
     *
     * 1. Thread cancellation is disabled for as long as the GIL is held. Without this a
     *    pthread_cancel() (ThreadBase::cancel_thread(), used during shutdown) can destroy the
     *    thread between Ensure and Release, leaving the GIL owned by a thread that no longer
     *    exists. Nobody can ever reacquire it after that. A cancel requested while
     *    cancellation is disabled simply stays pending and is delivered once the GIL has been
     *    released, which is exactly the behaviour we want.
     *
     * 2. The GIL is released even if an exception unwinds out of the scope. Trick's
     *    exec_terminate() throws Trick::ExecutiveException from inside PyRun_SimpleString
     *    (VariableServerSessionThread_loop.cpp catches it), and the previous straight-line
     *    code skipped PyGILState_Release() on that path, leaking the GIL.
     *
     * 3. active_parse_count tracks how many threads are in the interpreter, so that
     *    IPPython::shutdown() can tell whether reacquiring the GIL is safe or would block
     *    forever.
     */
    class GILGuard
    {
        public:
            GILGuard()
            {
                pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_cancel_state);
                // Incremented before Ensure() so that a thread blocked *waiting* for the GIL
                // is counted too. Erring towards "someone is in the interpreter" is the safe
                // direction: the worst case is that we skip Py_Finalize() unnecessarily.
                ++active_parse_count;
                gstate = PyGILState_Ensure();
            }

            ~GILGuard()
            {
                PyGILState_Release(gstate);
                --active_parse_count;
                pthread_setcancelstate(old_cancel_state, NULL);
            }

            GILGuard(const GILGuard&)            = delete;
            GILGuard& operator=(const GILGuard&) = delete;

        private:
            PyGILState_STATE gstate;
            int old_cancel_state;
    };

}

/**
@details
-# Loops through all of the memorymanager allocations testing if a name handle was given.
 -# If a name and a user type_name were given to the allocation
  -# If the user_type_name is not a Trick core class, prefixed with "Trick::"
  -# Create a python statement to assign the python name to an address: <name> = trick.castAsTYPE(int(<address>))
  -# Run the statement in the python interpreter
*/
void Trick::IPPython::get_TMM_named_variables() {
    //std::cout << "top level names at initialization" << std::endl ;
    Trick::ALLOC_INFO_MAP_ITER aim_it ;
    for ( aim_it = trick_MM->alloc_info_map_begin() ; aim_it != trick_MM->alloc_info_map_end() ; ++aim_it ) {
        ALLOC_INFO * alloc_info = (*aim_it).second ;
        if ( alloc_info->name != NULL and alloc_info->user_type_name != NULL ) {
            std::stringstream ss ;
            std::string user_type_name = alloc_info->user_type_name ;
            size_t start_colon ;
            while ( ( start_colon = user_type_name.find("::") ) != std::string::npos ) {
                user_type_name.replace( start_colon , 2 , "__" ) ;
            }
            // The castAs method may not exist if the class was hidden from SWIG (#ifndef SWIG).
            // Use a try/except block to test if the method exists or not.  If it doesn't exist
            // don't worry about it.  Also only assign python variable if it is pointing to
            // something python doesn't owns.  Otherwise we could free the object we're trying to assign.
            ss << "try:" << std::endl ;
            ss << "    if '" << alloc_info->name << "' not in globals() or " ;
            ss << alloc_info->name << ".thisown == False:" <<  std::endl ;
            ss << "        " << alloc_info->name << " = " ;
            ss << "trick.castAs" << user_type_name << "(int(" << alloc_info->start << "))" << std::endl ;
            ss << "except AttributeError:" << std::endl ;
            ss << "    pass" << std::endl ;
            GILGuard gil_guard;
            PyRun_SimpleString(ss.str().c_str());
        }
    }
}

//Initialize and run the Python input processor on the user input file.
int Trick::IPPython::init() {
    /** @par Detailed Design: */

    FILE *input_fp ;
    int ret ;
    std::string error_message ;

    // Run Py_Initialze first for python 2.x
#if PY_VERSION_HEX < 0x03000000
    Py_Initialize();
#endif

    /* Run the Swig generated routine in S_source_wrap.cpp. */
    init_swig_modules() ;

    // Run Py_Initialze after init_swig_modules for python 3.x
#if PY_VERSION_HEX >= 0x03000000
    Py_Initialize();
#endif

    // The following PyRun_ calls do not require the PyGILState guards because no threads are launched
    /* Import simulation specific routines into interpreter. */
    PyRun_SimpleString(
     "import sys\n"
     "import os\n"
     "import struct\n"
     "import binascii\n"
     "if 'VIRTUAL_ENV' in os.environ:\n"
     "    sys.path.append(os.path.join(os.environ['VIRTUAL_ENV'], \"lib\", f\"python{sys.version_info.major}.{sys.version_info.minor}\", \"site-packages\"))\n"
     "sys.path.append(os.getcwd() + '/trick.zip')\n"
     "sys.path.append(os.path.join(os.environ['TRICK_HOME'], 'share/trick/pymods'))\n"
     "sys.path += map(str.strip, os.environ['TRICK_PYTHON_PATH'].split(':'))\n"
     "import trick\n"
     "sys.path.append(os.getcwd() + \"/Modified_data\")\n"
    ) ;

    /* Make shortcut names for all known sim_objects. */
    get_TMM_named_variables() ;

    /* An input file is not required, if the name is empty just return. */
    if ( input_file.empty() ) {
        return(0) ;
    }

    if ((input_fp = fopen(input_file.c_str(), "r")) == NULL) {
        error_message = "No input file found named " + input_file ;
        exec_terminate_with_return(-1 , __FILE__ , __LINE__ , error_message.c_str() ) ;
    }

    /* Read and parse the input file. */
    if ( verify_input ) {
        PyRun_SimpleString("sys.settrace(trick.traceit)") ;
    }

    /* Read and parse the input file. */
    if ( save_input ) {
        PyRun_SimpleString("trick.open_input_file_log()") ;
        PyRun_SimpleString("sys.settrace(trick.traceittofile)") ;
    }

    if ( (ret = PyRun_SimpleFile(input_fp, input_file.c_str())) !=  0 ) {
        exec_terminate_with_return(ret , __FILE__ , __LINE__ , "Input Processor error\n" ) ;
    }

    if ( verify_input ) {
       std::stringstream ss ;
       ss << "import hashlib" << std::endl ;
       ss << "input_file = " << "'" << input_file.c_str() << "'" << std::endl;
       ss << "print('{0} SHA1: {1}'.format(input_file,hashlib.sha1(open(input_file, 'rb').read()).hexdigest()))" << std::endl ;
       PyRun_SimpleString(ss.str().c_str()) ;
       exec_terminate_with_return(ret , __FILE__ , __LINE__ , "Input file verification complete\n" ) ;
    }

    if ( save_input ) {
        PyRun_SimpleString("trick.close_input_file_log()") ;
        PyRun_SimpleString("sys.settrace(None)") ;
    }

    fclose(input_fp) ;

    // Release the GIL from the main thread.
    Py_UNBLOCK_THREADS

    return(0) ;
}

//Command to parse the given string.
int Trick::IPPython::parse(std::string in_string) {

    int ret ;
    in_string += "\n" ;

    GILGuard gil_guard;
    ret = PyRun_SimpleString(in_string.c_str());

    return ret ;

}

/**
 @details
 The incoming statement is assumed to be a conditional fragment, i.e. "a > b".  We need
 to get the return value of this fragment by setting the return value of it to a known
 variable name in the input processor.  We can then assign that return value to the
 incoming return_value reference.

-# Lock the input processor mutex
-# Create a complete statement that assigns the conditional fragment to our return value
-# parse the condition
-# copy the return value to the incoming cond_return_value
-# Unlock the input processor mutex
*/
int Trick::IPPython::parse_condition(std::string in_string, int & cond_return_val ) {

    in_string =  std::string("trick_ip.ip.return_val = ") + in_string + "\n" ;
    // Running the simple string will set return_val.
    int py_ret;
    {
        GILGuard gil_guard;
        py_ret = PyRun_SimpleString(in_string.c_str());
    }
    cond_return_val = return_val ;

    return py_ret ;

}

//Restart job that reloads event_list from checkpointable structures
int Trick::IPPython::restart() {
    /* Make shortcut names for all known sim_objects. */
    get_TMM_named_variables() ;
    return 0 ;
}

int Trick::IPPython::shutdown() {

    if ( Py_IsInitialized() ) {
        // If any thread is still inside parse()/parse_condition() then it owns the GIL,
        // or is queued for it, and may never give it back -- the usual cause is a
        // variable server command that blocks inside a model call, since SWIG does not
        // release the GIL around wrapped calls. Reacquiring the GIL below would then
        // block forever.
        //
        // Skipping Py_Finalize() in that case is safe. The process is about to exit, so
        // the OS reclaims the interpreter's memory regardless; all we give up is Python's
        // orderly teardown (atexit handlers, __del__ methods). That is a far better
        // outcome than wedging the sim, and it lets Executive::shutdown() carry on and
        // report the real return code instead of the run appearing to hang.
        int in_interpreter = active_parse_count;
        if (in_interpreter > 0)
        {
            message_publish(
                MSG_WARNING,
                "Trick::IPPython::shutdown() skipping Py_Finalize(): %d thread(s) still executing in the Python "
                "interpreter and holding the GIL. The Python interpreter will not be finalized.\n",
                in_interpreter);
            return (0);
        }

        // Obtain the GIL so that we can shut down properly
        // Check if the thread state is actually saved before trying to restore it
        // Python thread state is NULL when using JIT Input
        if (_save != NULL)
        {
            Py_BLOCK_THREADS _save = NULL;
        }
        Py_Finalize();
    }
    return(0) ;
}

1	/*
2	PURPOSE: ( Python input processor )
3	REFERENCE: ( Trick Simulation Environment )
4	ASSUMPTIONS AND LIMITATIONS: ( None )
5	CLASS: ( N/A )
6	LIBRARY DEPENDENCY: ( None )
7	PROGRAMMERS: ( Alex Lin NASA 2009 )
8	*/
9
10	#include <Python.h>
11
12	#include "trick/IPPython.hh"
13
14	#include "trick/MemoryManager.hh"
15	#include "trick/exec_proto.h"
16	#include "trick/io_alloc.h"
17	#include "trick/message_proto.h"
18	#include "trick/message_type.h"
19
20	#include <atomic>
21	#include <cstdio>
22	#include <cstdlib>
23	#include <cstring>
24	#include <iostream>
25	#include <pthread.h>
26	#include <sstream>
27	#include <string>
28	#include <unistd.h>
29
30	Trick::IPPython * the_pip ;
31
32	//Constructor
33	Trick::IPPython::IPPython() { the_pip = this; }	188✔
34
35	// Need to save the state of the main thread to allow child threads to run PyRun variants.
36	static PyThreadState *_save = NULL;
37
38	// Number of threads currently inside parse()/parse_condition(), i.e. threads that hold
39	// the Python GIL or are waiting to acquire it. See GILGuard and shutdown() below.
40	static std::atomic<int> active_parse_count(0);
41
42	namespace
43	{
44	/**
45	* Holds the Python GIL for a scope, with three guarantees that the bare
46	* PyGILState_Ensure()/PyGILState_Release() pair does not provide.
47	*
48	* 1. Thread cancellation is disabled for as long as the GIL is held. Without this a
49	* pthread_cancel() (ThreadBase::cancel_thread(), used during shutdown) can destroy the
50	* thread between Ensure and Release, leaving the GIL owned by a thread that no longer
51	* exists. Nobody can ever reacquire it after that. A cancel requested while
52	* cancellation is disabled simply stays pending and is delivered once the GIL has been
53	* released, which is exactly the behaviour we want.
54	*
55	* 2. The GIL is released even if an exception unwinds out of the scope. Trick's
56	* exec_terminate() throws Trick::ExecutiveException from inside PyRun_SimpleString
57	* (VariableServerSessionThread_loop.cpp catches it), and the previous straight-line
58	* code skipped PyGILState_Release() on that path, leaking the GIL.
59	*
60	* 3. active_parse_count tracks how many threads are in the interpreter, so that
61	* IPPython::shutdown() can tell whether reacquiring the GIL is safe or would block
62	* forever.
63	*/
64	class GILGuard
65	{
66	public:
67	GILGuard()	5,707✔
68	{	5,707✔
69	pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_cancel_state);	5,707✔
70	// Incremented before Ensure() so that a thread blocked waiting for the GIL
71	// is counted too. Erring towards "someone is in the interpreter" is the safe
72	// direction: the worst case is that we skip Py_Finalize() unnecessarily.
73	++active_parse_count;	5,707✔
74	gstate = PyGILState_Ensure();	5,707✔
75	}	5,707✔
76
77	~GILGuard()	5,707✔
78	{
79	PyGILState_Release(gstate);	5,707✔
80	--active_parse_count;	5,707✔
81	pthread_setcancelstate(old_cancel_state, NULL);	5,707✔
82	}	5,707✔
83
84	GILGuard(const GILGuard&) = delete;
85	GILGuard& operator=(const GILGuard&) = delete;
86
87	private:
88	PyGILState_STATE gstate;
89	int old_cancel_state;
90	};
91
92	}
93
94	/**
95	@details
96	-# Loops through all of the memorymanager allocations testing if a name handle was given.
97	-# If a name and a user type_name were given to the allocation
98	-# If the user_type_name is not a Trick core class, prefixed with "Trick::"
99	-# Create a python statement to assign the python name to an address: <name> = trick.castAsTYPE(int(<address>))
100	-# Run the statement in the python interpreter
101	*/
102	void Trick::IPPython::get_TMM_named_variables() {	199✔
103	//std::cout << "top level names at initialization" << std::endl ;
104	Trick::ALLOC_INFO_MAP_ITER aim_it ;	199✔
105	for ( aim_it = trick_MM->alloc_info_map_begin() ; aim_it != trick_MM->alloc_info_map_end() ; ++aim_it ) {	5,580✔
106	ALLOC_INFO * alloc_info = (*aim_it).second ;	5,381✔
107	if ( alloc_info->name != NULL and alloc_info->user_type_name != NULL ) {	5,381✔
108	std::stringstream ss ;	5,130✔
109	std::string user_type_name = alloc_info->user_type_name ;	5,130✔
110	size_t start_colon ;
111	while ( ( start_colon = user_type_name.find("::") ) != std::string::npos ) {	5,741✔
112	user_type_name.replace( start_colon , 2 , "__" ) ;	611✔
113	}
114	// The castAs method may not exist if the class was hidden from SWIG (#ifndef SWIG).
115	// Use a try/except block to test if the method exists or not. If it doesn't exist
116	// don't worry about it. Also only assign python variable if it is pointing to
117	// something python doesn't owns. Otherwise we could free the object we're trying to assign.
118	ss << "try:" << std::endl ;	5,130✔
119	ss << " if '" << alloc_info->name << "' not in globals() or " ;	5,130✔
120	ss << alloc_info->name << ".thisown == False:" << std::endl ;	5,130✔
121	ss << " " << alloc_info->name << " = " ;	5,130✔
122	ss << "trick.castAs" << user_type_name << "(int(" << alloc_info->start << "))" << std::endl ;	5,130✔
123	ss << "except AttributeError:" << std::endl ;	5,130✔
124	ss << " pass" << std::endl ;	5,130✔
125	GILGuard gil_guard;	5,130✔
126	PyRun_SimpleString(ss.str().c_str());	5,130✔
127	}	5,130✔
128	}
129	}	199✔
130
131	//Initialize and run the Python input processor on the user input file.
132	int Trick::IPPython::init() {	188✔
133	/** @par Detailed Design: */
134
135	FILE *input_fp ;
136	int ret ;
137	std::string error_message ;	188✔
138
139	// Run Py_Initialze first for python 2.x
140	#if PY_VERSION_HEX < 0x03000000
141	Py_Initialize();
142	#endif
143
144	/* Run the Swig generated routine in S_source_wrap.cpp. */
145	init_swig_modules() ;	188✔
146
147	// Run Py_Initialze after init_swig_modules for python 3.x
148	#if PY_VERSION_HEX >= 0x03000000
149	Py_Initialize();	188✔
150	#endif
151
152	// The following PyRun_ calls do not require the PyGILState guards because no threads are launched
153	/* Import simulation specific routines into interpreter. */
154	PyRun_SimpleString(	188✔
155	"import sys\n"
156	"import os\n"
157	"import struct\n"
158	"import binascii\n"
159	"if 'VIRTUAL_ENV' in os.environ:\n"
160	" sys.path.append(os.path.join(os.environ['VIRTUAL_ENV'], \"lib\", f\"python{sys.version_info.major}.{sys.version_info.minor}\", \"site-packages\"))\n"
161	"sys.path.append(os.getcwd() + '/trick.zip')\n"
162	"sys.path.append(os.path.join(os.environ['TRICK_HOME'], 'share/trick/pymods'))\n"
163	"sys.path += map(str.strip, os.environ['TRICK_PYTHON_PATH'].split(':'))\n"
164	"import trick\n"
165	"sys.path.append(os.getcwd() + \"/Modified_data\")\n"
166	) ;
167
168	/* Make shortcut names for all known sim_objects. */
169	get_TMM_named_variables() ;	188✔
170
171	/* An input file is not required, if the name is empty just return. */
172	if ( input_file.empty() ) {	188✔
173	return(0) ;	×
174	}
175
176	if ((input_fp = fopen(input_file.c_str(), "r")) == NULL) {	188✔
177	error_message = "No input file found named " + input_file ;	×
178	exec_terminate_with_return(-1 , __FILE__ , __LINE__ , error_message.c_str() ) ;	×
179	}
180
181	/* Read and parse the input file. */
182	if ( verify_input ) {	188✔
183	PyRun_SimpleString("sys.settrace(trick.traceit)") ;	×
184	}
185
186	/* Read and parse the input file. */
187	if ( save_input ) {	188✔
188	PyRun_SimpleString("trick.open_input_file_log()") ;	×
189	PyRun_SimpleString("sys.settrace(trick.traceittofile)") ;	×
190	}
191
192	if ( (ret = PyRun_SimpleFile(input_fp, input_file.c_str())) != 0 ) {	188✔
193	exec_terminate_with_return(ret , __FILE__ , __LINE__ , "Input Processor error\n" ) ;	×
194	}
195
196	if ( verify_input ) {	186✔
197	std::stringstream ss ;	×
198	ss << "import hashlib" << std::endl ;	×
199	ss << "input_file = " << "'" << input_file.c_str() << "'" << std::endl;	×
200	ss << "print('{0} SHA1: {1}'.format(input_file,hashlib.sha1(open(input_file, 'rb').read()).hexdigest()))" << std::endl ;	×
201	PyRun_SimpleString(ss.str().c_str()) ;	×
202	exec_terminate_with_return(ret , __FILE__ , __LINE__ , "Input file verification complete\n" ) ;	×
203	}	×
204
205	if ( save_input ) {	186✔
206	PyRun_SimpleString("trick.close_input_file_log()") ;	×
207	PyRun_SimpleString("sys.settrace(None)") ;	×
208	}
209
210	fclose(input_fp) ;	186✔
211
212	// Release the GIL from the main thread.
213	Py_UNBLOCK_THREADS	186✔
214
215	return(0) ;	186✔
216	}	188✔
217
218	//Command to parse the given string.
219	int Trick::IPPython::parse(std::string in_string) {	521✔
220
221	int ret ;
222	in_string += "\n" ;	521✔
223
224	GILGuard gil_guard;	521✔
225	ret = PyRun_SimpleString(in_string.c_str());	521✔
226
227	return ret ;	521✔
228
229	}	521✔
230
231	/**
232	@details
233	The incoming statement is assumed to be a conditional fragment, i.e. "a > b". We need
234	to get the return value of this fragment by setting the return value of it to a known
235	variable name in the input processor. We can then assign that return value to the
236	incoming return_value reference.
237
238	-# Lock the input processor mutex
239	-# Create a complete statement that assigns the conditional fragment to our return value
240	-# parse the condition
241	-# copy the return value to the incoming cond_return_value
242	-# Unlock the input processor mutex
243	*/
244	int Trick::IPPython::parse_condition(std::string in_string, int & cond_return_val ) {	56✔
245
246	in_string = std::string("trick_ip.ip.return_val = ") + in_string + "\n" ;	56✔
247	// Running the simple string will set return_val.
248	int py_ret;
249	{
250	GILGuard gil_guard;	56✔
251	py_ret = PyRun_SimpleString(in_string.c_str());	56✔
252	}	56✔
253	cond_return_val = return_val ;	56✔
254
255	return py_ret ;	56✔
256
257	}
258
259	//Restart job that reloads event_list from checkpointable structures
260	int Trick::IPPython::restart() {	11✔
261	/* Make shortcut names for all known sim_objects. */
262	get_TMM_named_variables() ;	11✔
263	return 0 ;	11✔
264	}
265
266	int Trick::IPPython::shutdown() {	155✔
267
268	if ( Py_IsInitialized() ) {	155✔
269	// If any thread is still inside parse()/parse_condition() then it owns the GIL,
270	// or is queued for it, and may never give it back -- the usual cause is a
271	// variable server command that blocks inside a model call, since SWIG does not
272	// release the GIL around wrapped calls. Reacquiring the GIL below would then
273	// block forever.
274	//
275	// Skipping Py_Finalize() in that case is safe. The process is about to exit, so
276	// the OS reclaims the interpreter's memory regardless; all we give up is Python's
277	// orderly teardown (atexit handlers, __del__ methods). That is a far better
278	// outcome than wedging the sim, and it lets Executive::shutdown() carry on and
279	// report the real return code instead of the run appearing to hang.
280	int in_interpreter = active_parse_count;	155✔
281	if (in_interpreter > 0)	155✔
282	{
NEW 283	message_publish(	×
284	MSG_WARNING,
285	"Trick::IPPython::shutdown() skipping Py_Finalize(): %d thread(s) still executing in the Python "
286	"interpreter and holding the GIL. The Python interpreter will not be finalized.\n",
287	in_interpreter);
NEW 288	return (0);	×
289	}
290
291	// Obtain the GIL so that we can shut down properly
292	// Check if the thread state is actually saved before trying to restore it
293	// Python thread state is NULL when using JIT Input
294	if (_save != NULL)	155✔
295	{
296	Py_BLOCK_THREADS _save = NULL;	155✔
297	}
298	Py_Finalize();	155✔
299	}
300	return(0) ;	155✔
301	}

nasa / trick / 30672691247

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous