3575403894

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/MilesCranmer/PySR/commit/<a class=hub.com/MilesCranmer/PySR/commit/d12a61c2fb34fc5e597191f75f256ea182f12399">d12a61c2f<a href="https://github.com/MilesCranmer/PySR/commit/d12a61c2fb34fc5e597191f75f256ea182f12399">">Merge </a><a class="double-link" href="https://github.com/MilesCranmer/PySR/commit/<a class="double-link" href="https://github.com/MilesCranmer/PySR/commit/dc2fef561cb691850436d87d66fe07310f96d014">dc2fef561</a>">dc2fef561</a><a href="https://github.com/MilesCranmer/PySR/commit/d12a61c2fb34fc5e597191f75f256ea182f12399"> into 31ffc0ec3">31ffc0ec3</a>

Pull Request Pull Request #230: Package compiler

Run Details

36 of 58 new or added lines in 5 files covered. (62.07%)

919 of 1042 relevant lines covered (88.2%)

0.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

74.45

/pysr/julia_helpers.py

"""Functions for initializing the Julia environment and installing deps."""
import sys
import subprocess
import warnings
from pathlib import Path
import os
from julia.api import JuliaError

from .version import __version__, __symbolic_regression_jl_version__

juliainfo = None
julia_initialized = False
julia_kwargs_at_initialization = None
julia_activated_env = None


def _load_juliainfo():
    """Execute julia.core.JuliaInfo.load(), and store as juliainfo."""
    global juliainfo

    if juliainfo is None:
        from julia.core import JuliaInfo

        try:
            juliainfo = JuliaInfo.load(julia="julia")
        except FileNotFoundError:
            env_path = os.environ["PATH"]
            raise FileNotFoundError(
                f"Julia is not installed in your PATH. Please install Julia and add it to your PATH.\n\nCurrent PATH: {env_path}",
            )

    return juliainfo


def _get_julia_project_dir():
    # Assumes it is in JULIA_PROJECT:
    assert "JULIA_PROJECT" in os.environ and os.environ["JULIA_PROJECT"] != ""
    try:
        cmds = [
            "julia",
            "--compile=min",
            "--startup-file=no",
            "-O0",
            "-g0",
            "-e import Pkg: project; print(project().path)",
        ]
        julia_project_dir_str = subprocess.run(
            cmds,
            capture_output=True,
            env=os.environ,
        ).stdout.decode()
    except FileNotFoundError:
        env_path = os.environ["PATH"]
        raise FileNotFoundError(
            f"Julia is not installed in your PATH. Please install Julia and add it to your PATH.\n\nCurrent PATH: {env_path}",
        )
    return Path(julia_project_dir_str).parent


def _get_julia_env_dir():
    # Have to manually get env dir:
    try:
        cmds = [
            "julia",
            "--compile=min",
            "--startup-file=no",
            "-O0",
            "-g0",
            "-e import Pkg: envdir; print(envdir())",
        ]
        julia_env_dir_str = subprocess.run(
            cmds,
            capture_output=True,
            env=os.environ,
        ).stdout.decode()
    except FileNotFoundError:
        env_path = os.environ["PATH"]
        raise FileNotFoundError(
            f"Julia is not installed in your PATH. Please install Julia and add it to your PATH.\n\nCurrent PATH: {env_path}",
        )
    return Path(julia_env_dir_str)


def _set_julia_project_env(julia_project, is_shared):
    if is_shared:
        if is_julia_version_greater_eq(version=(1, 7, 0)):
            os.environ["JULIA_PROJECT"] = "@" + str(julia_project)
        else:
            julia_env_dir = _get_julia_env_dir()
            os.environ["JULIA_PROJECT"] = str(julia_env_dir / julia_project)
    else:
        os.environ["JULIA_PROJECT"] = str(julia_project)


def _get_io_arg(quiet):
    io = "devnull" if quiet else "stderr"
    io_arg = f"io={io}" if is_julia_version_greater_eq(version=(1, 6, 0)) else ""
    return io_arg


def _import_error():
    return """
    Required dependencies are not installed or built.  Run the following code in the Python REPL:

        >>> import pysr
        >>> pysr.install()
    """


def _process_julia_project(julia_project):
    if julia_project is None:
        is_shared = True
        processed_julia_project = f"pysr-{__version__}"
    elif julia_project[0] == "@":
        is_shared = True
        processed_julia_project = julia_project[1:]
    else:
        is_shared = False
        processed_julia_project = Path(julia_project)
    return processed_julia_project, is_shared


def is_julia_version_greater_eq(juliainfo=None, version=(1, 6, 0)):
    """Check if Julia version is greater than specified version."""
    if juliainfo is None:
        juliainfo = _load_juliainfo()
    current_version = (
        juliainfo.version_major,
        juliainfo.version_minor,
        juliainfo.version_patch,
    )
    return current_version >= version


def _check_for_conflicting_libraries():  # pragma: no cover
    """Check whether there are conflicting modules, and display warnings."""
    # See https://github.com/pytorch/pytorch/issues/78829: importing
    # pytorch before running `pysr.fit` causes a segfault.
    torch_is_loaded = "torch" in sys.modules
    if torch_is_loaded:
        warnings.warn(
            "`torch` was loaded before the Julia instance started. "
            "This may cause a segfault when running `PySRRegressor.fit`. "
            "To avoid this, please run `pysr.julia_helpers.init_julia()` *before* "
            "importing `torch`. "
            "For updates, see https://github.com/pytorch/pytorch/issues/78829"
        )


def init_julia(
    julia_project=None,
    quiet=False,
    use_sysimage=True,
    sysimage_name=None,
    julia_kwargs=None,
):
    """Initialize julia binary, turning off compiled modules if needed."""
    global julia_initialized
    global julia_kwargs_at_initialization
    global julia_activated_env

    if not julia_initialized:
        _check_for_conflicting_libraries()

    _julia_version_assertion()
    processed_julia_project, is_shared = _process_julia_project(julia_project)
    _set_julia_project_env(processed_julia_project, is_shared)

    # TODO: Make checking optional.
    # Check if sysimage exists:
    if use_sysimage and sysimage_name is None and not julia_initialized:
        # TODO: Is there a faster way to get this dir?
        expected_sysimage = _get_julia_project_dir() / "pysr.so"
        # Check if this file exists:
        if expected_sysimage.exists():
            sysimage_name = str(expected_sysimage)

    if julia_kwargs is None:
        julia_kwargs = {"optimize": 3}

    if (
        sysimage_name is not None
        and "sysimage" not in julia_kwargs
        and not julia_initialized
    ):
        sysimage = str(sysimage_name)
        print(f"Found existing sysimage at {sysimage}. Loading.")
        julia_kwargs["sysimage"] = sysimage_name

    from julia.core import JuliaInfo, UnsupportedPythonError

    try:
        # TODO: Can we just get env info from this?
        info = JuliaInfo.load(julia="julia")
    except FileNotFoundError:
        env_path = os.environ["PATH"]
        raise FileNotFoundError(
            f"Julia is not installed in your PATH. Please install Julia and add it to your PATH.\n\nCurrent PATH: {env_path}",
        )

    if not info.is_pycall_built():
        raise ImportError(_import_error())

    from julia.core import Julia

    try:
        Julia(**julia_kwargs)
    except UnsupportedPythonError:
        # Static python binary, so we turn off pre-compiled modules.
        julia_kwargs = {**julia_kwargs, "compiled_modules": False}
        Julia(**julia_kwargs)

    from julia import Main as _Main

    Main = _Main

    if julia_activated_env is None:
        julia_activated_env = processed_julia_project

    if julia_initialized and julia_kwargs_at_initialization is not None:
        # Check if the kwargs are the same as the previous initialization
        init_set = set(julia_kwargs_at_initialization.items())
        new_set = set(julia_kwargs.items())
        set_diff = new_set - init_set
        # Remove the `compiled_modules` key, since it is not a user-specified kwarg:
        set_diff = {k: v for k, v in set_diff if k != "compiled_modules"}
        if len(set_diff) > 0:
            warnings.warn(
                "Julia has already started. The new Julia options "
                + str(set_diff)
                + " will be ignored."
            )

    if julia_initialized and julia_activated_env != processed_julia_project:
        Main.eval("using Pkg")

        io_arg = _get_io_arg(quiet)
        # Can't pass IO to Julia call as it evaluates to PyObject, so just directly
        # use Main.eval:
        Main.eval(
            f'Pkg.activate("{_escape_filename(processed_julia_project)}",'
            f"shared = Bool({int(is_shared)}), "
            f"{io_arg})"
        )

        julia_activated_env = processed_julia_project

    if not julia_initialized:
        julia_kwargs_at_initialization = julia_kwargs

    julia_initialized = True
    return Main


def _escape_filename(filename):
    """Turn a path into a string with correctly escaped backslashes."""
    str_repr = str(filename)
    str_repr = str_repr.replace("\\", "\\\\")
    return str_repr


def _julia_version_assertion():
    if not is_julia_version_greater_eq(version=(1, 6, 0)):
        raise NotImplementedError(
            "PySR requires Julia 1.6.0 or greater. "
            "Please update your Julia installation."
        )


def _backend_version_assertion(Main):
    try:
        backend_version = Main.eval("string(SymbolicRegression.PACKAGE_VERSION)")
        expected_backend_version = __symbolic_regression_jl_version__
        if backend_version != expected_backend_version:  # pragma: no cover
            warnings.warn(
                f"PySR backend (SymbolicRegression.jl) version {backend_version} "
                f"does not match expected version {expected_backend_version}. "
                "Things may break. "
                "Please update your PySR installation with "
                "`python -c 'import pysr; pysr.install()'`."
            )
    except JuliaError:  # pragma: no cover
        warnings.warn(
            "You seem to have an outdated version of SymbolicRegression.jl. "
            "Things may break. "
            "Please update your PySR installation with "
            "`python -c 'import pysr; pysr.install()'`."
        )


def _load_cluster_manager(Main, cluster_manager):
    Main.eval(f"import ClusterManagers: addprocs_{cluster_manager}")
    return Main.eval(f"addprocs_{cluster_manager}")


def _load_backend(Main):
    try:
        # Load namespace, so that various internal operators work:
        Main.eval("using SymbolicRegression")
    except (JuliaError, RuntimeError) as e:
        raise ImportError(_import_error()) from e

    _backend_version_assertion(Main)

    # Load Julia package SymbolicRegression.jl
    from julia import SymbolicRegression

    return SymbolicRegression

1	"""Functions for initializing the Julia environment and installing deps."""
2	import sys	1✔
3	import subprocess	1✔
4	import warnings	1✔
5	from pathlib import Path	1✔
6	import os	1✔
7	from julia.api import JuliaError	1✔
8
9	from .version import __version__, __symbolic_regression_jl_version__	1✔
10
11	juliainfo = None	1✔
12	julia_initialized = False	1✔
13	julia_kwargs_at_initialization = None	1✔
14	julia_activated_env = None	1✔
15
16
17	def _load_juliainfo():	1✔
18	"""Execute julia.core.JuliaInfo.load(), and store as juliainfo."""
19	global juliainfo
20
21	if juliainfo is None:	1✔
22	from julia.core import JuliaInfo	1✔
23
24	try:	1✔
25	juliainfo = JuliaInfo.load(julia="julia")	1✔
26	except FileNotFoundError:	×
27	env_path = os.environ["PATH"]	×
28	raise FileNotFoundError(	×
29	f"Julia is not installed in your PATH. Please install Julia and add it to your PATH.\n\nCurrent PATH: {env_path}",
30	)
31
32	return juliainfo	1✔
33
34
35	def _get_julia_project_dir():	1✔
36	# Assumes it is in JULIA_PROJECT:
37	assert "JULIA_PROJECT" in os.environ and os.environ["JULIA_PROJECT"] != ""	1✔
38	try:	1✔
39	cmds = [	1✔
40	"julia",
41	"--compile=min",
42	"--startup-file=no",
43	"-O0",
44	"-g0",
45	"-e import Pkg: project; print(project().path)",
46	]
47	julia_project_dir_str = subprocess.run(	1✔
48	cmds,
49	capture_output=True,
50	env=os.environ,
51	).stdout.decode()
NEW 52	except FileNotFoundError:	×
NEW 53	env_path = os.environ["PATH"]	×
NEW 54	raise FileNotFoundError(	×
55	f"Julia is not installed in your PATH. Please install Julia and add it to your PATH.\n\nCurrent PATH: {env_path}",
56	)
57	return Path(julia_project_dir_str).parent	1✔
58
59
60	def _get_julia_env_dir():	1✔
61	# Have to manually get env dir:
62	try:	×
NEW 63	cmds = [	×
64	"julia",
65	"--compile=min",
66	"--startup-file=no",
67	"-O0",
68	"-g0",
69	"-e import Pkg: envdir; print(envdir())",
70	]
71	julia_env_dir_str = subprocess.run(	×
72	cmds,
73	capture_output=True,
74	env=os.environ,
75	).stdout.decode()
76	except FileNotFoundError:	×
77	env_path = os.environ["PATH"]	×
78	raise FileNotFoundError(	×
79	f"Julia is not installed in your PATH. Please install Julia and add it to your PATH.\n\nCurrent PATH: {env_path}",
80	)
81	return Path(julia_env_dir_str)	×
82
83
84	def _set_julia_project_env(julia_project, is_shared):	1✔
85	if is_shared:	1✔
86	if is_julia_version_greater_eq(version=(1, 7, 0)):	1✔
87	os.environ["JULIA_PROJECT"] = "@" + str(julia_project)	1✔
88	else:
89	julia_env_dir = _get_julia_env_dir()	×
90	os.environ["JULIA_PROJECT"] = str(julia_env_dir / julia_project)	×
91	else:
92	os.environ["JULIA_PROJECT"] = str(julia_project)	×
93
94
95	def _get_io_arg(quiet):	1✔
96	io = "devnull" if quiet else "stderr"	1✔
97	io_arg = f"io={io}" if is_julia_version_greater_eq(version=(1, 6, 0)) else ""	1✔
98	return io_arg	1✔
99
100
101	def _import_error():	1✔
102	return """	×
103	Required dependencies are not installed or built. Run the following code in the Python REPL:
104
105	>>> import pysr
106	>>> pysr.install()
107	"""
108
109
110	def _process_julia_project(julia_project):	1✔
111	if julia_project is None:	1✔
112	is_shared = True	1✔
113	processed_julia_project = f"pysr-{__version__}"	1✔
114	elif julia_project[0] == "@":	1✔
115	is_shared = True	1✔
116	processed_julia_project = julia_project[1:]	1✔
117	else:
118	is_shared = False	×
119	processed_julia_project = Path(julia_project)	×
120	return processed_julia_project, is_shared	1✔
121
122
123	def is_julia_version_greater_eq(juliainfo=None, version=(1, 6, 0)):	1✔
124	"""Check if Julia version is greater than specified version."""
125	if juliainfo is None:	1✔
126	juliainfo = _load_juliainfo()	1✔
127	current_version = (	1✔
128	juliainfo.version_major,
129	juliainfo.version_minor,
130	juliainfo.version_patch,
131	)
132	return current_version >= version	1✔
133
134
135	def _check_for_conflicting_libraries(): # pragma: no cover
136	"""Check whether there are conflicting modules, and display warnings."""
137	# See https://github.com/pytorch/pytorch/issues/78829: importing
138	# pytorch before running `pysr.fit` causes a segfault.
139	torch_is_loaded = "torch" in sys.modules
140	if torch_is_loaded:
141	warnings.warn(
142	"`torch` was loaded before the Julia instance started. "
143	"This may cause a segfault when running `PySRRegressor.fit`. "
144	"To avoid this, please run `pysr.julia_helpers.init_julia()` before "
145	"importing `torch`. "
146	"For updates, see https://github.com/pytorch/pytorch/issues/78829"
147	)
148
149
150	def init_julia(	1✔
151	julia_project=None,
152	quiet=False,
153	use_sysimage=True,
154	sysimage_name=None,
155	julia_kwargs=None,
156	):
157	"""Initialize julia binary, turning off compiled modules if needed."""
158	global julia_initialized
159	global julia_kwargs_at_initialization
160	global julia_activated_env
161
162	if not julia_initialized:	1✔
163	_check_for_conflicting_libraries()	1✔
164
165	_julia_version_assertion()	1✔
166	processed_julia_project, is_shared = _process_julia_project(julia_project)	1✔
167	_set_julia_project_env(processed_julia_project, is_shared)	1✔
168
169	# TODO: Make checking optional.
170	# Check if sysimage exists:
171	if use_sysimage and sysimage_name is None and not julia_initialized:	1✔
172	# TODO: Is there a faster way to get this dir?
173	expected_sysimage = _get_julia_project_dir() / "pysr.so"	1✔
174	# Check if this file exists:
175	if expected_sysimage.exists():	1✔
NEW 176	sysimage_name = str(expected_sysimage)	×
177
178	if julia_kwargs is None:	1✔
179	julia_kwargs = {"optimize": 3}	1✔
180
181	if (	1✔
182	sysimage_name is not None
183	and "sysimage" not in julia_kwargs
184	and not julia_initialized
185	):
NEW 186	sysimage = str(sysimage_name)	×
NEW 187	print(f"Found existing sysimage at {sysimage}. Loading.")	×
NEW 188	julia_kwargs["sysimage"] = sysimage_name	×
189
190	from julia.core import JuliaInfo, UnsupportedPythonError	1✔
191
192	try:	1✔
193	# TODO: Can we just get env info from this?
194	info = JuliaInfo.load(julia="julia")	1✔
195	except FileNotFoundError:	×
196	env_path = os.environ["PATH"]	×
197	raise FileNotFoundError(	×
198	f"Julia is not installed in your PATH. Please install Julia and add it to your PATH.\n\nCurrent PATH: {env_path}",
199	)
200
201	if not info.is_pycall_built():	1✔
202	raise ImportError(_import_error())	×
203
204	from julia.core import Julia	1✔
205
206	try:	1✔
207	Julia(**julia_kwargs)	1✔
208	except UnsupportedPythonError:	×
209	# Static python binary, so we turn off pre-compiled modules.
210	julia_kwargs = {**julia_kwargs, "compiled_modules": False}	×
211	Julia(**julia_kwargs)	×
212
213	from julia import Main as _Main	1✔
214
215	Main = _Main	1✔
216
217	if julia_activated_env is None:	1✔
218	julia_activated_env = processed_julia_project	1✔
219
220	if julia_initialized and julia_kwargs_at_initialization is not None:	1✔
221	# Check if the kwargs are the same as the previous initialization
222	init_set = set(julia_kwargs_at_initialization.items())	1✔
223	new_set = set(julia_kwargs.items())	1✔
224	set_diff = new_set - init_set	1✔
225	# Remove the `compiled_modules` key, since it is not a user-specified kwarg:
226	set_diff = {k: v for k, v in set_diff if k != "compiled_modules"}	1✔
227	if len(set_diff) > 0:	1✔
228	warnings.warn(	1✔
229	"Julia has already started. The new Julia options "
230	+ str(set_diff)
231	+ " will be ignored."
232	)
233
234	if julia_initialized and julia_activated_env != processed_julia_project:	1✔
235	Main.eval("using Pkg")	1✔
236
237	io_arg = _get_io_arg(quiet)	1✔
238	# Can't pass IO to Julia call as it evaluates to PyObject, so just directly
239	# use Main.eval:
240	Main.eval(	1✔
241	f'Pkg.activate("{_escape_filename(processed_julia_project)}",'
242	f"shared = Bool({int(is_shared)}), "
243	f"{io_arg})"
244	)
245
246	julia_activated_env = processed_julia_project	1✔
247
248	if not julia_initialized:	1✔
249	julia_kwargs_at_initialization = julia_kwargs	1✔
250
251	julia_initialized = True	1✔
252	return Main	1✔
253
254
255	def _escape_filename(filename):	1✔
256	"""Turn a path into a string with correctly escaped backslashes."""
257	str_repr = str(filename)	1✔
258	str_repr = str_repr.replace("\\", "\\\\")	1✔
259	return str_repr	1✔
260
261
262	def _julia_version_assertion():	1✔
263	if not is_julia_version_greater_eq(version=(1, 6, 0)):	1✔
264	raise NotImplementedError(	×
265	"PySR requires Julia 1.6.0 or greater. "
266	"Please update your Julia installation."
267	)
268
269
270	def _backend_version_assertion(Main):	1✔
271	try:	1✔
272	backend_version = Main.eval("string(SymbolicRegression.PACKAGE_VERSION)")	1✔
273	expected_backend_version = __symbolic_regression_jl_version__	1✔
274	if backend_version != expected_backend_version: # pragma: no cover
275	warnings.warn(
276	f"PySR backend (SymbolicRegression.jl) version {backend_version} "
277	f"does not match expected version {expected_backend_version}. "
278	"Things may break. "
279	"Please update your PySR installation with "
280	"`python -c 'import pysr; pysr.install()'`."
281	)
282	except JuliaError: # pragma: no cover
283	warnings.warn(
284	"You seem to have an outdated version of SymbolicRegression.jl. "
285	"Things may break. "
286	"Please update your PySR installation with "
287	"`python -c 'import pysr; pysr.install()'`."
288	)
289
290
291	def _load_cluster_manager(Main, cluster_manager):	1✔
292	Main.eval(f"import ClusterManagers: addprocs_{cluster_manager}")	×
293	return Main.eval(f"addprocs_{cluster_manager}")	×
294
295
296	def _load_backend(Main):	1✔
297	try:	1✔
298	# Load namespace, so that various internal operators work:
299	Main.eval("using SymbolicRegression")	1✔
300	except (JuliaError, RuntimeError) as e:	×
301	raise ImportError(_import_error()) from e	×
302
303	_backend_version_assertion(Main)	1✔
304
305	# Load Julia package SymbolicRegression.jl
306	from julia import SymbolicRegression	1✔
307
308	return SymbolicRegression	1✔

MilesCranmer / PySR / 3575403894

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous