20704884474

Committed 05 Jan 2026 04:14AM UTC coverage: 67.869% (-0.1%) from 67.988%

Build # 20704884474

Build Type

push

github

Committed by

jtmoon79

Commit Message

(TEST) fix test_PyEventReader_new_asl_odl_panic

Run Details

15705 of 23140 relevant lines covered (67.87%)

118323.14 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.67

/src/python/venv.rs

// src/python/venv.rs

//! Create and manage the Python virtual environment for `s4`.

#[allow(deprecated)]
use std::env::home_dir;
use std::fs::{
    create_dir_all,
    remove_dir_all,
};
use std::io::{
    ErrorKind,
    Error,
    Result,
};
use std::path::PathBuf;
use std::vec;

use ::include_dir::{
    include_dir,
    Dir as Include_Dir,
};
use ::regex::bytes::Regex;
#[allow(unused_imports)]
use ::si_trace_print::{
    defñ,
    defn,
    defo,
    defx,
    def1ñ,
    def1n,
    def1o,
    def1x,
};
use ::tempfile::{
    TempDir,
    env::temp_dir,
};
use ::version_compare::{
    compare_to,
    Cmp,
};

use crate::{
    debug_panic,
    e_err,
    e_wrn,
};
#[allow(unused_imports)]
use crate::de_err;
use crate::common::{
    Bytes,
    Result3E,
};
use crate::python::pyrunner::{
    ChunkDelimiter,
    PipeSz,
    PyRunner,
    PythonToUse,
    RECV_TIMEOUT,
};

/// Minimum acceptable Python version, checked during venv creation
const PYTHON_VERSION_MIN: &str = "3.9";

/// pipe size for PyRunner instances used in venv creation
const PIPE_SZ: PipeSz = 16384;

/// chunk delimiter for PyRunner instances used during venv creation
const CHUNK_DELIMITER: ChunkDelimiter = b'\n';

/// only for user-facing help messages.
/// XXX: this must match the path used in `venv_path()`
pub const PYTHON_VENV_PATH_DEFAULT: &str = "~/.config/s4/venv";

/// Python project name
const PROJECT_NAME: &str = "s4_event_readers";

/// embedded files of the s4_event_readers Python project.
/// unpacked and installed during venv creation
static PY_PROJECT_DIR: Include_Dir = include_dir!("$CARGO_MANIFEST_DIR/src/python/s4_event_readers");

/// return path to python s4 venv directory.
/// does not check if it exists
pub fn venv_path() -> PathBuf {
    #[allow(deprecated)]
    let mut home: PathBuf = match home_dir() {
        Some(h) => h,
        None => {
            // TODO: what is a better fallback path?
            temp_dir()
        },
    };
    home.push(".config");
    home.push("s4");
    home.push("venv");

    if cfg!(test) {
        // for tests, use a temporary path
        home = temp_dir();
        home.push("tmp-s4-test-python-venv");
    }

    defñ!("return {:?}", home);

    home
}

/// copy the python project into a temporary directory
/// for build and installation
pub fn deploy_pyproject_s4_event_readers() -> Result<TempDir> {
    defn!();

    let tmpdir: TempDir = match TempDir::with_prefix(format!("{}_", PROJECT_NAME)) {
        Ok(td) => td,
        Err(err) => {
            defx!("TempDir::new() error: {}", err);
            return Result::Err(err);
        }
    };

    defo!("PY_PROJECT_DIR.extract({:?})", tmpdir.path());
    match PY_PROJECT_DIR.extract(tmpdir.path()) {
        Ok(_) => {}
        Err(err) => {
            defx!("dir.extract error: {}", err);
            return Result::Err(err);
        }
    }
    defx!("Extracted PY_PROJECT_DIR {:?}", PY_PROJECT_DIR.path());

    Result::Ok(tmpdir)
}

/// extract and compare the version
/// return `Ok` if version is acceptable
/// `data` is the output of `python --version`, e.g. `b'Python 3.9.7\n'`
pub(crate) fn extract_compare_version(data: &Bytes) -> Result<()> {
    def1n!();
    // create regex to extract version
    let version_re: Regex = match Regex::new(r"^Python (\d+)\.(\d+)\.(\d+)") {
        Ok(re) => re,
        Err(err) => {
            def1x!("Regex::new returned Err {:?}", err);
            return Err(
                Error::new(
                    ErrorKind::Other,
                    format!("failed to create python version regex; {}", err),
                )
            );
        }
    };
    // regex capture the data
    let captures = match version_re.captures(data) {
        Some(captures) => captures,
        None => {
            def1x!("version_re.captures returned None");
            return Err(
                Error::new(
                    ErrorKind::Other,
                    format!("failed to capture python version from output {:?}", data),
                )
            );
        }
    };
    // get the captured part as a String
    let version_str: String = match std::str::from_utf8(&captures[0]) {
        Ok(s) => {
            def1o!("Converted version capture to str: {:?}", s);
            // remove "Python " prefix
            let mut s: String = s.to_string();
            s = s.replace("Python ", "");
            def1o!("Extracted version string: {:?}", s);

            s
        }
        Err(err) => {
            def1x!("from_utf8 returned Err {:?}", err);
            return Err(
                Error::new(
                    ErrorKind::Other,
                    format!("failed to convert python version capture to str; {}", err),
                )
            );
        }
    };
    def1o!("Found Python version {}", version_str);
    // compare the version strings with `compare_to()`
    match compare_to(&version_str, PYTHON_VERSION_MIN, Cmp::Ge) {
        Ok(cmp_result) => {
            if cmp_result {
                def1o!("Python version {} is acceptable", version_str);
            } else {
                def1x!("Python version too low; return Unsupported");
                return Err(
                    Error::new(
                        ErrorKind::Unsupported,
                        format!("python version {} is less than the required minimum {}", version_str, PYTHON_VERSION_MIN),
                    )
                );
            }
        }
        Err(err) => {
            def1x!("compare_to returned Err {:?}", err);
            return Err(
                Error::new(
                    ErrorKind::Other,
                    format!("failed to compare python versions {:?}", err),
                )
            );
        }
    }
    def1x!("return Ok");

    Ok(())
}

/// create the Python virtual environment using [`PyRunner`]s
pub fn create() -> Result3E<()> {
    def1n!();

    // run `python --version` to sanity check Python
    // using found Python interpreter
    // TODO: warn if version is less than required minimum
    let mut pyrunner = match PyRunner::new(
        PythonToUse::EnvPath,
        PIPE_SZ,
        RECV_TIMEOUT,
        Some(CHUNK_DELIMITER),
        None,
        None,
        vec![
        "--version",
    ]) {
        Ok(pyrunner) => pyrunner,
        Err(err) => {
            de_err!("Failed to create first Python runner: {}", err);
            def1x!("Python --version; return Err {:?}", err);
            return Result3E::Err(err);
        }
    };
    match pyrunner.run(true, true, true) {
        Ok((stdout, _stderr)) => {
             match extract_compare_version(&stdout) {
                Ok(_) => {},
                Err(err) if err.kind() == ErrorKind::Unsupported => {
                    e_wrn!("{}", err.to_string());
                }
                Err(err) => {
                    e_err!("Failed to compare python version: {}", err);
                    def1x!("pyrunner.run() returned Err {:?}", err);
                    return Result3E::ErrNoReprint(err);
                }
            }
        }
        Err(err) => {
            e_err!("Failed to run python --version: {}", err);
            def1x!("pyrunner.run() returned Err {:?}", err);
            return Result3E::ErrNoReprint(err);
        }
    }
    // remember the python path used
    let python_path = pyrunner.python_path;

    // rm the prior venv
    let venv_path_pb: PathBuf = venv_path();
    if venv_path_pb.exists() {
        def1o!("remove_dir_all({:?})", venv_path_pb);
        eprintln!("remove_dir_all({})\n", venv_path_pb.display());
        match remove_dir_all(venv_path_pb.as_path()) {
            Result::Ok(_) => {},
            Result::Err(err) => {
                e_err!("Failed to remove virtual environment directory {:?}: {}", venv_path_pb, err);
                def1x!("remove_dir_all returned {:?}", err);
                return Result3E::ErrNoReprint(err);
            }
        }
    }

    // create the venv directory including parent directories
    // using found Python interpreter
    def1o!("create_dir_all({:?})", venv_path_pb);
    eprintln!("create_dir_all({})\n", venv_path_pb.display());
    match create_dir_all(venv_path_pb.as_path()) {
        Result::Ok(_) => {},
        Result::Err(err) => {
            e_err!("Failed to create virtual environment directory {:?}: {}", venv_path_pb, err);
            def1x!("create_dir_all returned {:?}", err);
            return Result3E::ErrNoReprint(err);
        }
    }

    // one more sanity check
    if ! venv_path_pb.is_dir() {
        let err_msg = format!("Python virtual environment path {:?} is not a directory", venv_path_pb);
        e_err!("{}", err_msg);
        def1x!("{}", err_msg);
        return Result3E::ErrNoReprint(Error::new(ErrorKind::NotADirectory, err_msg));
    }

    // create the venv using found Python interpreter
    let venv_path_s: &str = match venv_path_pb.as_os_str().to_str() {
        Some(s) => s,
        None => {
            def1x!("failed convert path to os_str to str {:?}, return Unsupported", venv_path_pb);
            return Result3E::Err(
                Error::new(
                    ErrorKind::Unsupported,
                    format!("failed to convert path to os_str to str; {:?}", venv_path_pb),
                )
            );
        }
    };
    match PyRunner::run_once(
        PythonToUse::Value,
        PIPE_SZ,
        RECV_TIMEOUT,
        CHUNK_DELIMITER,
        Some(python_path),
        vec![
            "-m",
            "venv",
            "--clear",
            "--copies",
            "--prompt",
            "s4",
            venv_path_s,
        ],
        true,
    ) {
        Ok(_) => {},
        Result::Err(err) => {
            e_err!("Failed to create Python virtual environment; venv command failed: {}", err);
            def1x!("pyrunner.run() returned {:?}", err);
            return Result3E::ErrNoReprint(err);
        }
    }

    // ensure pip is installed in the venv
    match PyRunner::run_once(
        PythonToUse::Venv,
        PIPE_SZ,
        RECV_TIMEOUT,
        CHUNK_DELIMITER,
        None,
        vec![
            "-m",
            "ensurepip",
        ],
        true,
    ) {
        Ok(_) => {},
        Err(err) => {
            e_err!("Failed to ensurepip: {}", err);
            def1x!("PyRunner::new failed {:?}", err);
            return Result3E::ErrNoReprint(err);
        }
    };

    // prevent pip from version checks
    match PyRunner::run_once(
        PythonToUse::Venv,
        PIPE_SZ,
        RECV_TIMEOUT,
        CHUNK_DELIMITER,
        None,
        vec![
            "-m",
            "pip",
            "config",
            "set",
            "--site",
            "global.disable-pip-version-check",
            "true",
        ],
        true,
    ) {
        Ok(_) => {},
        Err(err) => {
            e_err!("Failed to disable pip version check: {}", err);
            def1x!("PyRunner::new failed {:?}", err);
            return Result3E::ErrNoReprint(err);
        }
    };
    match PyRunner::run_once(
        PythonToUse::Venv,
        PIPE_SZ,
        RECV_TIMEOUT,
        CHUNK_DELIMITER,
        None,
        vec![
            "-m",
            "pip",
            "config",
            "set",
            "--site",
            "global.disable-python-version-warning",
            "true",
        ],
        true,
    ) {
        Ok(_) => {},
        Err(err) => {
            e_err!("Failed to disable python version warning: {}", err);
            def1x!("PyRunner::new failed {:?}", err);
            return Result3E::ErrNoReprint(err);
        }
    };

    // expand the project into a temporary directory
    let mut project_tmp_path: TempDir = match deploy_pyproject_s4_event_readers() {
        Ok(p) => p,
        Err(err) => {
            e_err!("Failed to deploy python project: {}", err);
            def1x!("deploy_pyproject_s4_event_readers failed {:?}", err);
            return Result3E::ErrNoReprint(err);
        }
    };
    if cfg!(debug_assertions) {
        project_tmp_path.disable_cleanup(true);
        def1o!("Temporary project remains at {:?}", project_tmp_path.path());
    }
    let project_tmp_path_s: &str = match project_tmp_path.path().as_os_str().to_str() {
        Some(s) => s,
        None => {
            let err_msg = format!(
                "failed to convert path to os_str to str; {:?}", project_tmp_path
            );
            e_err!("{}", err_msg);
            def1x!("{}", err_msg);
            return Result3E::Err(Error::new(ErrorKind::Other, err_msg));
        }
    };
    eprintln!(
        "inflated project {} to temporary path {:?}\n", PROJECT_NAME, project_tmp_path.path()
    );

    // install wheel
    // this is purely to workaround using an older etl-parser package
    // that uses legacy setup.py installation. without wheel
    // the later project installation warns of a deprecated install method.
    match PyRunner::run_once(
        PythonToUse::Venv,
        PIPE_SZ,
        RECV_TIMEOUT,
        CHUNK_DELIMITER,
        None,
        vec![
            "-m",
            "pip",
            "install",
            "wheel",
        ],
        true,
    ) {
        Ok(_) => {},
        Err(err) => {
            e_err!("Failed to ensurepip: {}", err);
            def1x!("PyRunner::new failed {:?}", err);
            return Result3E::ErrNoReprint(err);
        }
    };

    // install required python packages
    match PyRunner::run_once(
        PythonToUse::Venv,
        PIPE_SZ,
        RECV_TIMEOUT,
        CHUNK_DELIMITER,
        None,
        vec![
            "-m",
            "pip",
            "install",
            project_tmp_path_s,
        ],
        true,
    ) {
        Ok(_) => {},
        Err(err) => {
            e_err!("Failed to install python packages: {}", err);
            def1x!("PyRunner::new failed {:?}", err);
            return Result3E::ErrNoReprint(err);
        }
    };

    // get site-packages path
    let site_path: Bytes = match PyRunner::run_once(
        PythonToUse::Venv,
        PIPE_SZ,
        RECV_TIMEOUT,
        CHUNK_DELIMITER,
        None,
        vec![
            "-c",
            "import sysconfig; print(sysconfig.get_path(\"purelib\"))",
        ],
        true,
    ) {
        Ok((_, stdout, _)) => stdout,
        Err(err) => {
            e_wrn!("Failed to get site-packages path: {}", err);
            debug_panic!("PyRunner::run_once failed {:?}", err);

            Bytes::with_capacity(0)
        }
    };

    let mut argv = vec![
        "-m",
        "compileall",
        "-o2",
    ];
    // add site-packages path if it was obtained
    if ! site_path.is_empty() {
        let site_path_s: &str = match std::str::from_utf8(&site_path) {
            Ok(s) => s.trim(),
            Err(err) => {
                let err_msg = format!(
                    "failed to convert site-packages path to str; path {:?}, error {}",
                    site_path, err
                );
                e_err!("{}", err_msg);
                debug_panic!("from_utf8 failed {:?}", err);
                return Result3E::ErrNoReprint(Error::new(ErrorKind::Other, err_msg));
            }
        };
        argv.push(site_path_s);
    }

    // precompile installed site-packages
    match PyRunner::run_once(
        PythonToUse::Venv,
        PIPE_SZ,
        RECV_TIMEOUT,
        CHUNK_DELIMITER,
        None,
        argv,
        true,
    ) {
        Ok(_) => {},
        Err(err) => {
            e_wrn!("Failed to precompile python site-packages: {}; hopefully this can be ignored.", err);
            debug_panic!("PyRunner::run_once failed {:?}", err);
        }
    };

    if let Err(err) = PyRunner::run_once(
        PythonToUse::Venv,
        PIPE_SZ,
        RECV_TIMEOUT,
        CHUNK_DELIMITER,
        None,
        vec![
            "-OO",
            "-m",
            "s4_event_readers",
        ],
        true,
    ) {
        e_err!("Failed to run s4_event_readers module test: {}", err);
        def1x!("PyRunner::new failed {:?}", err);
        return Result3E::ErrNoReprint(err);
    }

    // touch special flag file to mark the venv is fully created
    let flag_path: PathBuf = venv_path().join("done");
    if let Err(err) = std::fs::write(&flag_path, b"created by s4") {
        e_err!("Failed to create {:?}: {}", flag_path, err);
        def1x!("std::fs::write returned {:?}", err);
        return Result3E::ErrNoReprint(err);
    }

    eprintln!("Python virtual environment created at {}", venv_path_pb.display());
    eprintln!("This environment will be automatically used by s4 for Python-based event readers, i.e. for .asl, .etl, .odl files.");

    def1x!("return Ok");

    Result3E::Ok(())
}

1	// src/python/venv.rs
2
3	//! Create and manage the Python virtual environment for `s4`.
4
5	#[allow(deprecated)]
6	use std::env::home_dir;
7	use std::fs::{
8	create_dir_all,
9	remove_dir_all,
10	};
11	use std::io::{
12	ErrorKind,
13	Error,
14	Result,
15	};
16	use std::path::PathBuf;
17	use std::vec;
18
19	use ::include_dir::{
20	include_dir,
21	Dir as Include_Dir,
22	};
23	use ::regex::bytes::Regex;
24	#[allow(unused_imports)]
25	use ::si_trace_print::{
26	defñ,
27	defn,
28	defo,
29	defx,
30	def1ñ,
31	def1n,
32	def1o,
33	def1x,
34	};
35	use ::tempfile::{
36	TempDir,
37	env::temp_dir,
38	};
39	use ::version_compare::{
40	compare_to,
41	Cmp,
42	};
43
44	use crate::{
45	debug_panic,
46	e_err,
47	e_wrn,
48	};
49	#[allow(unused_imports)]
50	use crate::de_err;
51	use crate::common::{
52	Bytes,
53	Result3E,
54	};
55	use crate::python::pyrunner::{
56	ChunkDelimiter,
57	PipeSz,
58	PyRunner,
59	PythonToUse,
60	RECV_TIMEOUT,
61	};
62
63	/// Minimum acceptable Python version, checked during venv creation
64	const PYTHON_VERSION_MIN: &str = "3.9";
65
66	/// pipe size for PyRunner instances used in venv creation
67	const PIPE_SZ: PipeSz = 16384;
68
69	/// chunk delimiter for PyRunner instances used during venv creation
70	const CHUNK_DELIMITER: ChunkDelimiter = b'\n';
71
72	/// only for user-facing help messages.
73	/// XXX: this must match the path used in `venv_path()`
74	pub const PYTHON_VENV_PATH_DEFAULT: &str = "~/.config/s4/venv";
75
76	/// Python project name
77	const PROJECT_NAME: &str = "s4_event_readers";
78
79	/// embedded files of the s4_event_readers Python project.
80	/// unpacked and installed during venv creation
81	static PY_PROJECT_DIR: Include_Dir = include_dir!("$CARGO_MANIFEST_DIR/src/python/s4_event_readers");
82
83	/// return path to python s4 venv directory.
84	/// does not check if it exists
85	pub fn venv_path() -> PathBuf {	4✔
86	#[allow(deprecated)]
87	let mut home: PathBuf = match home_dir() {	4✔
88	Some(h) => h,	4✔
89	None => {
90	// TODO: what is a better fallback path?
91	temp_dir()	×
92	},
93	};
94	home.push(".config");	4✔
95	home.push("s4");	4✔
96	home.push("venv");	4✔
97
98	if cfg!(test) {	4✔
99	// for tests, use a temporary path	4✔
100	home = temp_dir();	4✔
101	home.push("tmp-s4-test-python-venv");	4✔
102	}	4✔
103
104	defñ!("return {:?}", home);	4✔
105
106	home	4✔
107	}	4✔
108
109	/// copy the python project into a temporary directory
110	/// for build and installation
111	pub fn deploy_pyproject_s4_event_readers() -> Result<TempDir> {	2✔
112	defn!();	2✔
113
114	let tmpdir: TempDir = match TempDir::with_prefix(format!("{}_", PROJECT_NAME)) {	2✔
115	Ok(td) => td,	2✔
116	Err(err) => {	×
117	defx!("TempDir::new() error: {}", err);	×
118	return Result::Err(err);	×
119	}
120	};
121
122	defo!("PY_PROJECT_DIR.extract({:?})", tmpdir.path());	2✔
123	match PY_PROJECT_DIR.extract(tmpdir.path()) {	2✔
124	Ok(_) => {}	2✔
125	Err(err) => {	×
126	defx!("dir.extract error: {}", err);	×
127	return Result::Err(err);	×
128	}
129	}
130	defx!("Extracted PY_PROJECT_DIR {:?}", PY_PROJECT_DIR.path());	2✔
131
132	Result::Ok(tmpdir)	2✔
133	}	2✔
134
135	/// extract and compare the version
136	/// return `Ok` if version is acceptable
137	/// `data` is the output of `python --version`, e.g. `b'Python 3.9.7\n'`
138	pub(crate) fn extract_compare_version(data: &Bytes) -> Result<()> {	4✔
139	def1n!();	4✔
140	// create regex to extract version
141	let version_re: Regex = match Regex::new(r"^Python (\d+)\.(\d+)\.(\d+)") {	4✔
142	Ok(re) => re,	4✔
143	Err(err) => {	×
144	def1x!("Regex::new returned Err {:?}", err);	×
145	return Err(	×
146	Error::new(	×
147	ErrorKind::Other,	×
148	format!("failed to create python version regex; {}", err),	×
149	)	×
150	);	×
151	}
152	};
153	// regex capture the data
154	let captures = match version_re.captures(data) {	4✔
155	Some(captures) => captures,	3✔
156	None => {
157	def1x!("version_re.captures returned None");	1✔
158	return Err(	1✔
159	Error::new(	1✔
160	ErrorKind::Other,	1✔
161	format!("failed to capture python version from output {:?}", data),	1✔
162	)	1✔
163	);	1✔
164	}
165	};
166	// get the captured part as a String
167	let version_str: String = match std::str::from_utf8(&captures[0]) {	3✔
168	Ok(s) => {	3✔
169	def1o!("Converted version capture to str: {:?}", s);	3✔
170	// remove "Python " prefix
171	let mut s: String = s.to_string();	3✔
172	s = s.replace("Python ", "");	3✔
173	def1o!("Extracted version string: {:?}", s);	3✔
174
175	s	3✔
176	}
177	Err(err) => {	×
178	def1x!("from_utf8 returned Err {:?}", err);	×
179	return Err(	×
180	Error::new(	×
181	ErrorKind::Other,	×
182	format!("failed to convert python version capture to str; {}", err),	×
183	)	×
184	);	×
185	}
186	};
187	def1o!("Found Python version {}", version_str);	3✔
188	// compare the version strings with `compare_to()`
189	match compare_to(&version_str, PYTHON_VERSION_MIN, Cmp::Ge) {	3✔
190	Ok(cmp_result) => {	3✔
191	if cmp_result {	3✔
192	def1o!("Python version {} is acceptable", version_str);	2✔
193	} else {
194	def1x!("Python version too low; return Unsupported");	1✔
195	return Err(	1✔
196	Error::new(	1✔
197	ErrorKind::Unsupported,	1✔
198	format!("python version {} is less than the required minimum {}", version_str, PYTHON_VERSION_MIN),	1✔
199	)	1✔
200	);	1✔
201	}
202	}
203	Err(err) => {	×
204	def1x!("compare_to returned Err {:?}", err);	×
205	return Err(	×
206	Error::new(	×
207	ErrorKind::Other,	×
208	format!("failed to compare python versions {:?}", err),	×
209	)	×
210	);	×
211	}
212	}
213	def1x!("return Ok");	2✔
214
215	Ok(())	2✔
216	}	4✔
217
218	/// create the Python virtual environment using [`PyRunner`]s
219	pub fn create() -> Result3E<()> {	1✔
220	def1n!();	1✔
221
222	// run `python --version` to sanity check Python
223	// using found Python interpreter
224	// TODO: warn if version is less than required minimum
225	let mut pyrunner = match PyRunner::new(	1✔
226	PythonToUse::EnvPath,	1✔
227	PIPE_SZ,	1✔
228	RECV_TIMEOUT,	1✔
229	Some(CHUNK_DELIMITER),	1✔
230	None,	1✔
231	None,	1✔
232	vec![	1✔
233	"--version",	1✔
234	]) {	1✔
235	Ok(pyrunner) => pyrunner,	1✔
236	Err(err) => {	×
237	de_err!("Failed to create first Python runner: {}", err);	×
238	def1x!("Python --version; return Err {:?}", err);	×
239	return Result3E::Err(err);	×
240	}
241	};
242	match pyrunner.run(true, true, true) {	1✔
243	Ok((stdout, _stderr)) => {	1✔
244	match extract_compare_version(&stdout) {	1✔
245	Ok(_) => {},	1✔
246	Err(err) if err.kind() == ErrorKind::Unsupported => {	×
247	e_wrn!("{}", err.to_string());	×
248	}	×
249	Err(err) => {	×
250	e_err!("Failed to compare python version: {}", err);	×
251	def1x!("pyrunner.run() returned Err {:?}", err);	×
252	return Result3E::ErrNoReprint(err);	×
253	}
254	}
255	}
256	Err(err) => {	×
257	e_err!("Failed to run python --version: {}", err);	×
258	def1x!("pyrunner.run() returned Err {:?}", err);	×
259	return Result3E::ErrNoReprint(err);	×
260	}
261	}
262	// remember the python path used
263	let python_path = pyrunner.python_path;	1✔
264
265	// rm the prior venv
266	let venv_path_pb: PathBuf = venv_path();	1✔
267	if venv_path_pb.exists() {	1✔
268	def1o!("remove_dir_all({:?})", venv_path_pb);	×
269	eprintln!("remove_dir_all({})\n", venv_path_pb.display());	×
270	match remove_dir_all(venv_path_pb.as_path()) {	×
271	Result::Ok(_) => {},	×
272	Result::Err(err) => {	×
273	e_err!("Failed to remove virtual environment directory {:?}: {}", venv_path_pb, err);	×
274	def1x!("remove_dir_all returned {:?}", err);	×
275	return Result3E::ErrNoReprint(err);	×
276	}
277	}
278	}	1✔
279
280	// create the venv directory including parent directories
281	// using found Python interpreter
282	def1o!("create_dir_all({:?})", venv_path_pb);	1✔
283	eprintln!("create_dir_all({})\n", venv_path_pb.display());	1✔
284	match create_dir_all(venv_path_pb.as_path()) {	1✔
285	Result::Ok(_) => {},	1✔
286	Result::Err(err) => {	×
287	e_err!("Failed to create virtual environment directory {:?}: {}", venv_path_pb, err);	×
288	def1x!("create_dir_all returned {:?}", err);	×
289	return Result3E::ErrNoReprint(err);	×
290	}
291	}
292
293	// one more sanity check
294	if ! venv_path_pb.is_dir() {	1✔
295	let err_msg = format!("Python virtual environment path {:?} is not a directory", venv_path_pb);	×
296	e_err!("{}", err_msg);	×
297	def1x!("{}", err_msg);	×
298	return Result3E::ErrNoReprint(Error::new(ErrorKind::NotADirectory, err_msg));	×
299	}	1✔
300
301	// create the venv using found Python interpreter
302	let venv_path_s: &str = match venv_path_pb.as_os_str().to_str() {	1✔
303	Some(s) => s,	1✔
304	None => {
305	def1x!("failed convert path to os_str to str {:?}, return Unsupported", venv_path_pb);	×
306	return Result3E::Err(	×
307	Error::new(	×
308	ErrorKind::Unsupported,	×
309	format!("failed to convert path to os_str to str; {:?}", venv_path_pb),	×
310	)	×
311	);	×
312	}
313	};
314	match PyRunner::run_once(	1✔
315	PythonToUse::Value,	1✔
316	PIPE_SZ,	1✔
317	RECV_TIMEOUT,	1✔
318	CHUNK_DELIMITER,	1✔
319	Some(python_path),	1✔
320	vec![	1✔
321	"-m",	1✔
322	"venv",	1✔
323	"--clear",	1✔
324	"--copies",	1✔
325	"--prompt",	1✔
326	"s4",	1✔
327	venv_path_s,	1✔
328	],	1✔
329	true,	1✔
330	) {	1✔
331	Ok(_) => {},	1✔
332	Result::Err(err) => {	×
333	e_err!("Failed to create Python virtual environment; venv command failed: {}", err);	×
334	def1x!("pyrunner.run() returned {:?}", err);	×
335	return Result3E::ErrNoReprint(err);	×
336	}
337	}
338
339	// ensure pip is installed in the venv
340	match PyRunner::run_once(	1✔
341	PythonToUse::Venv,	1✔
342	PIPE_SZ,	1✔
343	RECV_TIMEOUT,	1✔
344	CHUNK_DELIMITER,	1✔
345	None,	1✔
346	vec![	1✔
347	"-m",	1✔
348	"ensurepip",	1✔
349	],	1✔
350	true,	1✔
351	) {	1✔
352	Ok(_) => {},	1✔
353	Err(err) => {	×
354	e_err!("Failed to ensurepip: {}", err);	×
355	def1x!("PyRunner::new failed {:?}", err);	×
356	return Result3E::ErrNoReprint(err);	×
357	}
358	};
359
360	// prevent pip from version checks
361	match PyRunner::run_once(	1✔
362	PythonToUse::Venv,	1✔
363	PIPE_SZ,	1✔
364	RECV_TIMEOUT,	1✔
365	CHUNK_DELIMITER,	1✔
366	None,	1✔
367	vec![	1✔
368	"-m",	1✔
369	"pip",	1✔
370	"config",	1✔
371	"set",	1✔
372	"--site",	1✔
373	"global.disable-pip-version-check",	1✔
374	"true",	1✔
375	],	1✔
376	true,	1✔
377	) {	1✔
378	Ok(_) => {},	1✔
379	Err(err) => {	×
380	e_err!("Failed to disable pip version check: {}", err);	×
381	def1x!("PyRunner::new failed {:?}", err);	×
382	return Result3E::ErrNoReprint(err);	×
383	}
384	};
385	match PyRunner::run_once(	1✔
386	PythonToUse::Venv,	1✔
387	PIPE_SZ,	1✔
388	RECV_TIMEOUT,	1✔
389	CHUNK_DELIMITER,	1✔
390	None,	1✔
391	vec![	1✔
392	"-m",	1✔
393	"pip",	1✔
394	"config",	1✔
395	"set",	1✔
396	"--site",	1✔
397	"global.disable-python-version-warning",	1✔
398	"true",	1✔
399	],	1✔
400	true,	1✔
401	) {	1✔
402	Ok(_) => {},	1✔
403	Err(err) => {	×
404	e_err!("Failed to disable python version warning: {}", err);	×
405	def1x!("PyRunner::new failed {:?}", err);	×
406	return Result3E::ErrNoReprint(err);	×
407	}
408	};
409
410	// expand the project into a temporary directory
411	let mut project_tmp_path: TempDir = match deploy_pyproject_s4_event_readers() {	1✔
412	Ok(p) => p,	1✔
413	Err(err) => {	×
414	e_err!("Failed to deploy python project: {}", err);	×
415	def1x!("deploy_pyproject_s4_event_readers failed {:?}", err);	×
416	return Result3E::ErrNoReprint(err);	×
417	}
418	};
419	if cfg!(debug_assertions) {	1✔
420	project_tmp_path.disable_cleanup(true);	1✔
421	def1o!("Temporary project remains at {:?}", project_tmp_path.path());	1✔
422	}	×
423	let project_tmp_path_s: &str = match project_tmp_path.path().as_os_str().to_str() {	1✔
424	Some(s) => s,	1✔
425	None => {
426	let err_msg = format!(	×
427	"failed to convert path to os_str to str; {:?}", project_tmp_path	×
428	);
429	e_err!("{}", err_msg);	×
430	def1x!("{}", err_msg);	×
431	return Result3E::Err(Error::new(ErrorKind::Other, err_msg));	×
432	}
433	};
434	eprintln!(	1✔
435	"inflated project {} to temporary path {:?}\n", PROJECT_NAME, project_tmp_path.path()	1✔
436	);
437
438	// install wheel
439	// this is purely to workaround using an older etl-parser package
440	// that uses legacy setup.py installation. without wheel
441	// the later project installation warns of a deprecated install method.
442	match PyRunner::run_once(	1✔
443	PythonToUse::Venv,	1✔
444	PIPE_SZ,	1✔
445	RECV_TIMEOUT,	1✔
446	CHUNK_DELIMITER,	1✔
447	None,	1✔
448	vec![	1✔
449	"-m",	1✔
450	"pip",	1✔
451	"install",	1✔
452	"wheel",	1✔
453	],	1✔
454	true,	1✔
455	) {	1✔
456	Ok(_) => {},	1✔
457	Err(err) => {	×
458	e_err!("Failed to ensurepip: {}", err);	×
459	def1x!("PyRunner::new failed {:?}", err);	×
460	return Result3E::ErrNoReprint(err);	×
461	}
462	};
463
464	// install required python packages
465	match PyRunner::run_once(	1✔
466	PythonToUse::Venv,	1✔
467	PIPE_SZ,	1✔
468	RECV_TIMEOUT,	1✔
469	CHUNK_DELIMITER,	1✔
470	None,	1✔
471	vec![	1✔
472	"-m",	1✔
473	"pip",	1✔
474	"install",	1✔
475	project_tmp_path_s,	1✔
476	],	1✔
477	true,	1✔
478	) {	1✔
479	Ok(_) => {},	1✔
480	Err(err) => {	×
481	e_err!("Failed to install python packages: {}", err);	×
482	def1x!("PyRunner::new failed {:?}", err);	×
483	return Result3E::ErrNoReprint(err);	×
484	}
485	};
486
487	// get site-packages path
488	let site_path: Bytes = match PyRunner::run_once(	1✔
489	PythonToUse::Venv,	1✔
490	PIPE_SZ,	1✔
491	RECV_TIMEOUT,	1✔
492	CHUNK_DELIMITER,	1✔
493	None,	1✔
494	vec![	1✔
495	"-c",	1✔
496	"import sysconfig; print(sysconfig.get_path(\"purelib\"))",	1✔
497	],	1✔
498	true,	1✔
499	) {	1✔
500	Ok((_, stdout, _)) => stdout,	1✔
501	Err(err) => {	×
502	e_wrn!("Failed to get site-packages path: {}", err);	×
503	debug_panic!("PyRunner::run_once failed {:?}", err);	×
504
505	Bytes::with_capacity(0)	×
506	}
507	};
508
509	let mut argv = vec![	1✔
510	"-m",
511	"compileall",	1✔
512	"-o2",	1✔
513	];
514	// add site-packages path if it was obtained
515	if ! site_path.is_empty() {	1✔
516	let site_path_s: &str = match std::str::from_utf8(&site_path) {	1✔
517	Ok(s) => s.trim(),	1✔
518	Err(err) => {	×
519	let err_msg = format!(	×
520	"failed to convert site-packages path to str; path {:?}, error {}",	×
521	site_path, err
522	);
523	e_err!("{}", err_msg);	×
524	debug_panic!("from_utf8 failed {:?}", err);	×
525	return Result3E::ErrNoReprint(Error::new(ErrorKind::Other, err_msg));	×
526	}
527	};
528	argv.push(site_path_s);	1✔
529	}	×
530
531	// precompile installed site-packages
532	match PyRunner::run_once(	1✔
533	PythonToUse::Venv,	1✔
534	PIPE_SZ,	1✔
535	RECV_TIMEOUT,	1✔
536	CHUNK_DELIMITER,	1✔
537	None,	1✔
538	argv,	1✔
539	true,	1✔
540	) {	1✔
541	Ok(_) => {},	1✔
542	Err(err) => {	×
543	e_wrn!("Failed to precompile python site-packages: {}; hopefully this can be ignored.", err);	×
544	debug_panic!("PyRunner::run_once failed {:?}", err);	×
545	}
546	};
547
548	if let Err(err) = PyRunner::run_once(	1✔
549	PythonToUse::Venv,	1✔
550	PIPE_SZ,	1✔
551	RECV_TIMEOUT,	1✔
552	CHUNK_DELIMITER,	1✔
553	None,	1✔
554	vec![	1✔
555	"-OO",	1✔
556	"-m",	1✔
557	"s4_event_readers",	1✔
558	],	1✔
559	true,	1✔
560	) {	1✔
561	e_err!("Failed to run s4_event_readers module test: {}", err);	×
562	def1x!("PyRunner::new failed {:?}", err);	×
563	return Result3E::ErrNoReprint(err);	×
564	}	1✔
565
566	// touch special flag file to mark the venv is fully created
567	let flag_path: PathBuf = venv_path().join("done");	1✔
568	if let Err(err) = std::fs::write(&flag_path, b"created by s4") {	1✔
569	e_err!("Failed to create {:?}: {}", flag_path, err);	×
570	def1x!("std::fs::write returned {:?}", err);	×
571	return Result3E::ErrNoReprint(err);	×
572	}	1✔
573
574	eprintln!("Python virtual environment created at {}", venv_path_pb.display());	1✔
575	eprintln!("This environment will be automatically used by s4 for Python-based event readers, i.e. for .asl, .etl, .odl files.");	1✔
576
577	def1x!("return Ok");	1✔
578
579	Result3E::Ok(())	1✔
580	}	1✔

jtmoon79 / super-speedy-syslog-searcher / 20704884474

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous