30326158672

Committed 28 Jul 2026 03:31AM UTC coverage: 70.857% (+1.9%) from 68.982%

Build # 30326158672

Build Type

push

github

Committed by

jtmoon79

Commit Message

(LIB) bump macro-string 0.3.0

Coverage Stats

19964 of 28175 relevant lines covered (70.86%)

1699356.33 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

67.21

/src/python/venv.rs

// src/python/venv.rs

//! Create and manage the Python virtual environment for `s4`.

#[allow(deprecated)]
use std::env::home_dir;
use std::fs::{
    create_dir_all,
    remove_dir_all,
};
use std::io::{
    ErrorKind,
    Error,
    Result,
};
use std::path::PathBuf;
use std::vec;

use ::include_dir::{
    include_dir,
    Dir as Include_Dir,
};
use ::regex::bytes::Regex;
#[allow(unused_imports)]
use ::si_trace_print::{
    defñ,
    defn,
    defo,
    defx,
    def1ñ,
    def1n,
    def1o,
    def1x,
};
use ::tempfile::{
    TempDir,
    env::temp_dir,
};
use ::version_compare::{
    compare_to,
    Cmp,
};

use crate::{
    debug_panic,
    e_err,
    e_wrn,
};
#[allow(unused_imports)]
use crate::de_err;
use crate::common::{
    Bytes,
    PathId,
    Result3E,
};
use crate::python::pyrunner::{
    ChunkDelimiter,
    PipeSz,
    PyRunner,
    PythonToUse,
    RECV_TIMEOUT,
};

/// Minimum acceptable Python version, checked during venv creation
const PYTHON_VERSION_MIN: &str = "3.9";

/// pipe size for PyRunner instances used in venv creation
const PIPE_SZ: PipeSz = 16384;

/// chunk delimiter for PyRunner instances used during venv creation
const CHUNK_DELIMITER: ChunkDelimiter = b'\n';

/// only for user-facing help messages.
/// XXX: this must match the path used in `venv_path()`
pub const PYTHON_VENV_PATH_DEFAULT: &str = "~/.config/s4/venv";

/// Python project name
const PROJECT_NAME: &str = "s4_event_readers";

/// embedded files of the s4_event_readers Python project.
/// unpacked and installed during venv creation
static PY_PROJECT_DIR: Include_Dir = include_dir!("$CARGO_MANIFEST_DIR/src/python/s4_event_readers");

/// return path to python s4 venv directory.
/// does not check if it exists
pub fn venv_path() -> PathBuf {
    #[allow(deprecated)]
    let mut home: PathBuf = match home_dir() {
        Some(h) => h,
        None => {
            // TODO: what is a better fallback path?
            temp_dir()
        },
    };
    home.push(".config");
    home.push("s4");
    home.push("venv");

    if cfg!(test) {
        // for tests, use a temporary path
        home = temp_dir();
        home.push("tmp-s4-test-python-venv");
    }

    defñ!("return {:?}", home);

    home
}

/// copy the python project into a temporary directory
/// for build and installation
pub fn deploy_pyproject_s4_event_readers() -> Result<TempDir> {
    defn!();

    let tmpdir: TempDir = match TempDir::with_prefix(format!("{}_", PROJECT_NAME)) {
        Ok(td) => td,
        Err(err) => {
            defx!("TempDir::new() error: {}", err);
            return Result::Err(err);
        }
    };

    defo!("PY_PROJECT_DIR.extract({:?})", tmpdir.path());
    match PY_PROJECT_DIR.extract(tmpdir.path()) {
        Ok(_) => {}
        Err(err) => {
            defx!("dir.extract error: {}", err);
            return Result::Err(err);
        }
    }
    defx!("Extracted PY_PROJECT_DIR {:?}", PY_PROJECT_DIR.path());

    Result::Ok(tmpdir)
}

/// extract and compare the version
/// return `Ok` if version is acceptable
/// `data` is the output of `python --version`, e.g. `b'Python 3.9.7\n'`
pub(crate) fn extract_compare_version(data: &Bytes) -> Result<()> {
    def1n!();
    // create regex to extract version
    let version_re: Regex = match Regex::new(r"^Python (\d+)\.(\d+)\.(\d+)") {
        Ok(re) => re,
        Err(err) => {
            def1x!("Regex::new returned Err {:?}", err);
            return Err(
                Error::new(
                    ErrorKind::Other,
                    format!("failed to create python version regex; {}", err),
                )
            );
        }
    };
    // regex capture the data
    let captures = match version_re.captures(data) {
        Some(captures) => captures,
        None => {
            def1x!("version_re.captures returned None");
            return Err(
                Error::new(
                    ErrorKind::Other,
                    format!("failed to capture python version from output {:?}", data),
                )
            );
        }
    };
    // get the captured part as a String
    let version_str: String = match std::str::from_utf8(&captures[0]) {
        Ok(s) => {
            def1o!("Converted version capture to str: {:?}", s);
            // remove "Python " prefix
            let mut s: String = s.to_string();
            s = s.replace("Python ", "");
            def1o!("Extracted version string: {:?}", s);

            s
        }
        Err(err) => {
            def1x!("from_utf8 returned Err {:?}", err);
            return Err(
                Error::new(
                    ErrorKind::Other,
                    format!("failed to convert python version capture to str; {}", err),
                )
            );
        }
    };
    def1o!("Found Python version {}", version_str);
    // compare the version strings with `compare_to()`
    match compare_to(&version_str, PYTHON_VERSION_MIN, Cmp::Ge) {
        Ok(cmp_result) => {
            if cmp_result {
                def1o!("Python version {} is acceptable", version_str);
            } else {
                def1x!("Python version too low; return Unsupported");
                return Err(
                    Error::new(
                        ErrorKind::Unsupported,
                        format!("python version {} is less than the required minimum {}", version_str, PYTHON_VERSION_MIN),
                    )
                );
            }
        }
        Err(err) => {
            def1x!("compare_to returned Err {:?}", err);
            return Err(
                Error::new(
                    ErrorKind::Other,
                    format!("failed to compare python versions {:?}", err),
                )
            );
        }
    }
    def1x!("return Ok");

    Ok(())
}

/// create the Python virtual environment using [`PyRunner`]s
pub fn create() -> Result3E<()> {
    def1n!();

    // run `python --version` to sanity check Python
    // using found Python interpreter
    // TODO: warn if version is less than required minimum
    let mut pyrunner = match PyRunner::new(
        PythonToUse::EnvPath,
        PathId::default(),
        PIPE_SZ,
        RECV_TIMEOUT,
        Some(CHUNK_DELIMITER),
        None,
        None,
        vec![
        "--version",
    ]) {
        Ok(pyrunner) => pyrunner,
        Err(err) => {
            de_err!("Failed to create first Python runner: {}", err);
            def1x!("Python --version; return Err {:?}", err);
            return Result3E::Err(err);
        }
    };
    match pyrunner.run(true, true, true) {
        Ok((stdout, _stderr)) => {
             match extract_compare_version(&stdout) {
                Ok(_) => {},
                Err(err) if err.kind() == ErrorKind::Unsupported => {
                    e_wrn!("{}", err.to_string());
                }
                Err(err) => {
                    e_err!("Failed to compare python version: {}", err);
                    def1x!("pyrunner.run() returned Err {:?}", err);
                    return Result3E::ErrNoReprint(err);
                }
            }
        }
        Err(err) => {
            e_err!("Failed to run python --version: {}", err);
            def1x!("pyrunner.run() returned Err {:?}", err);
            return Result3E::ErrNoReprint(err);
        }
    }
    // remember the python path used
    let python_path = pyrunner.python_path.clone();

    // rm the prior venv
    let venv_path_pb: PathBuf = venv_path();
    if venv_path_pb.exists() {
        def1o!("remove_dir_all({:?})", venv_path_pb);
        eprintln!("remove_dir_all({})\n", venv_path_pb.display());
        match remove_dir_all(venv_path_pb.as_path()) {
            Result::Ok(_) => {},
            Result::Err(err) => {
                e_err!("Failed to remove virtual environment directory {:?}: {}", venv_path_pb, err);
                def1x!("remove_dir_all returned {:?}", err);
                return Result3E::ErrNoReprint(err);
            }
        }
    }

    // create the venv directory including parent directories
    // using found Python interpreter
    def1o!("create_dir_all({:?})", venv_path_pb);
    eprintln!("create_dir_all({})\n", venv_path_pb.display());
    match create_dir_all(venv_path_pb.as_path()) {
        Result::Ok(_) => {},
        Result::Err(err) => {
            e_err!("Failed to create virtual environment directory {:?}: {}", venv_path_pb, err);
            def1x!("create_dir_all returned {:?}", err);
            return Result3E::ErrNoReprint(err);
        }
    }

    // one more sanity check
    if ! venv_path_pb.is_dir() {
        let err_msg = format!("Python virtual environment path {:?} is not a directory", venv_path_pb);
        e_err!("{}", err_msg);
        def1x!("{}", err_msg);
        return Result3E::ErrNoReprint(Error::new(ErrorKind::NotADirectory, err_msg));
    }

    // create the venv using found Python interpreter
    let venv_path_s: &str = match venv_path_pb.as_os_str().to_str() {
        Some(s) => s,
        None => {
            def1x!("failed convert path to os_str to str {:?}, return Unsupported", venv_path_pb);
            return Result3E::Err(
                Error::new(
                    ErrorKind::Unsupported,
                    format!("failed to convert path to os_str to str; {:?}", venv_path_pb),
                )
            );
        }
    };
    match PyRunner::run_once(
        PythonToUse::Value,
        PIPE_SZ,
        RECV_TIMEOUT,
        CHUNK_DELIMITER,
        Some(python_path.to_string()),
        vec![
            "-m",
            "venv",
            "--clear",
            "--prompt",
            "s4",
            venv_path_s,
        ],
        true,
    ) {
        Ok(_) => {},
        Result::Err(err) => {
            e_err!("Failed to create Python virtual environment; venv command failed: {}", err);
            def1x!("pyrunner.run() returned {:?}", err);
            return Result3E::ErrNoReprint(err);
        }
    }

    // ensure pip is installed in the venv
    match PyRunner::run_once(
        PythonToUse::Venv,
        PIPE_SZ,
        RECV_TIMEOUT,
        CHUNK_DELIMITER,
        None,
        vec![
            "-m",
            "ensurepip",
        ],
        true,
    ) {
        Ok(_) => {},
        Err(err) => {
            e_err!("Failed to ensurepip: {}", err);
            def1x!("PyRunner::new failed {:?}", err);
            return Result3E::ErrNoReprint(err);
        }
    };

    // prevent pip from version checks
    match PyRunner::run_once(
        PythonToUse::Venv,
        PIPE_SZ,
        RECV_TIMEOUT,
        CHUNK_DELIMITER,
        None,
        vec![
            "-m",
            "pip",
            "config",
            "set",
            "--site",
            "global.disable-pip-version-check",
            "true",
        ],
        true,
    ) {
        Ok(_) => {},
        Err(err) => {
            e_err!("Failed to disable pip version check: {}", err);
            def1x!("PyRunner::new failed {:?}", err);
            return Result3E::ErrNoReprint(err);
        }
    };
    match PyRunner::run_once(
        PythonToUse::Venv,
        PIPE_SZ,
        RECV_TIMEOUT,
        CHUNK_DELIMITER,
        None,
        vec![
            "-m",
            "pip",
            "config",
            "set",
            "--site",
            "global.disable-python-version-warning",
            "true",
        ],
        true,
    ) {
        Ok(_) => {},
        Err(err) => {
            e_err!("Failed to disable python version warning: {}", err);
            def1x!("PyRunner::new failed {:?}", err);
            return Result3E::ErrNoReprint(err);
        }
    };

    // expand the project into a temporary directory
    let mut project_tmp_path: TempDir = match deploy_pyproject_s4_event_readers() {
        Ok(p) => p,
        Err(err) => {
            e_err!("Failed to deploy python project: {}", err);
            def1x!("deploy_pyproject_s4_event_readers failed {:?}", err);
            return Result3E::ErrNoReprint(err);
        }
    };
    if cfg!(debug_assertions) {
        project_tmp_path.disable_cleanup(true);
        def1o!("Temporary project remains at {:?}", project_tmp_path.path());
    }
    let project_tmp_path_s: &str = match project_tmp_path.path().as_os_str().to_str() {
        Some(s) => s,
        None => {
            let err_msg = format!(
                "failed to convert path to os_str to str; {:?}", project_tmp_path
            );
            e_err!("{}", err_msg);
            def1x!("{}", err_msg);
            return Result3E::Err(Error::other(err_msg));
        }
    };
    eprintln!(
        "inflated project {} to temporary path {:?}\n", PROJECT_NAME, project_tmp_path.path()
    );

    // install wheel
    // this is purely to workaround using an older etl-parser package
    // that uses legacy setup.py installation. without wheel
    // the later project installation warns of a deprecated install method.
    match PyRunner::run_once(
        PythonToUse::Venv,
        PIPE_SZ,
        RECV_TIMEOUT,
        CHUNK_DELIMITER,
        None,
        vec![
            "-m",
            "pip",
            "install",
            "wheel",
        ],
        true,
    ) {
        Ok(_) => {},
        Err(err) => {
            e_err!("Failed to ensurepip: {}", err);
            def1x!("PyRunner::new failed {:?}", err);
            return Result3E::ErrNoReprint(err);
        }
    };

    // install required python packages
    match PyRunner::run_once(
        PythonToUse::Venv,
        PIPE_SZ,
        RECV_TIMEOUT,
        CHUNK_DELIMITER,
        None,
        vec![
            "-m",
            "pip",
            "install",
            project_tmp_path_s,
        ],
        true,
    ) {
        Ok(_) => {},
        Err(err) => {
            e_err!("Failed to install python packages: {}", err);
            def1x!("PyRunner::new failed {:?}", err);
            return Result3E::ErrNoReprint(err);
        }
    };

    // get site-packages path
    let site_path: Bytes = match PyRunner::run_once(
        PythonToUse::Venv,
        PIPE_SZ,
        RECV_TIMEOUT,
        CHUNK_DELIMITER,
        None,
        vec![
            "-c",
            "import sysconfig; print(sysconfig.get_path(\"purelib\"))",
        ],
        true,
    ) {
        Ok((_, stdout, _)) => stdout,
        Err(err) => {
            e_wrn!("Failed to get site-packages path: {}", err);
            debug_panic!("PyRunner::run_once failed {:?}", err);

            Bytes::with_capacity(0)
        }
    };

    let mut argv = vec![
        "-m",
        "compileall",
        "-o0",
        "-o1",
        "-o2",
    ];
    // add site-packages path if it was obtained
    if ! site_path.is_empty() {
        let site_path_s: &str = match std::str::from_utf8(&site_path) {
            Ok(s) => s.trim(),
            Err(err) => {
                let err_msg = format!(
                    "failed to convert site-packages path to str; path {:?}, error {}",
                    site_path, err
                );
                e_err!("{}", err_msg);
                debug_panic!("from_utf8 failed {:?}", err);
                return Result3E::ErrNoReprint(Error::new(ErrorKind::Other, err_msg));
            }
        };
        argv.push(site_path_s);
    }

    // precompile installed site-packages
    match PyRunner::run_once(
        PythonToUse::Venv,
        PIPE_SZ,
        RECV_TIMEOUT,
        CHUNK_DELIMITER,
        None,
        argv,
        true,
    ) {
        Ok(_) => {},
        Err(err) => {
            e_wrn!("Failed to precompile python site-packages: {}; hopefully this can be ignored.", err);
            debug_panic!("PyRunner::run_once failed {:?}", err);
        }
    };

    if let Err(err) = PyRunner::run_once(
        PythonToUse::Venv,
        PIPE_SZ,
        RECV_TIMEOUT,
        CHUNK_DELIMITER,
        None,
        vec![
            "-OO",
            "-m",
            "s4_event_readers",
        ],
        true,
    ) {
        e_err!("Failed to run s4_event_readers module test: {}", err);
        def1x!("PyRunner::new failed {:?}", err);
        return Result3E::ErrNoReprint(err);
    }

    // touch special flag file to mark the venv is fully created
    let flag_path: PathBuf = venv_path().join("done");
    if let Err(err) = std::fs::write(&flag_path, b"created by s4") {
        e_err!("Failed to create {:?}: {}", flag_path, err);
        def1x!("std::fs::write returned {:?}", err);
        return Result3E::ErrNoReprint(err);
    }

    eprintln!("Python virtual environment created at {}", venv_path_pb.display());
    eprintln!("This environment will be automatically used by s4 for Python-based event readers, i.e. for .asl, .etl, .odl files.");

    def1x!("return Ok");

    Result3E::Ok(())
}

1	// src/python/venv.rs
2
3	//! Create and manage the Python virtual environment for `s4`.
4
5	#[allow(deprecated)]
6	use std::env::home_dir;
7	use std::fs::{
8	create_dir_all,
9	remove_dir_all,
10	};
11	use std::io::{
12	ErrorKind,
13	Error,
14	Result,
15	};
16	use std::path::PathBuf;
17	use std::vec;
18
19	use ::include_dir::{
20	include_dir,
21	Dir as Include_Dir,
22	};
23	use ::regex::bytes::Regex;
24	#[allow(unused_imports)]
25	use ::si_trace_print::{
26	defñ,
27	defn,
28	defo,
29	defx,
30	def1ñ,
31	def1n,
32	def1o,
33	def1x,
34	};
35	use ::tempfile::{
36	TempDir,
37	env::temp_dir,
38	};
39	use ::version_compare::{
40	compare_to,
41	Cmp,
42	};
43
44	use crate::{
45	debug_panic,
46	e_err,
47	e_wrn,
48	};
49	#[allow(unused_imports)]
50	use crate::de_err;
51	use crate::common::{
52	Bytes,
53	PathId,
54	Result3E,
55	};
56	use crate::python::pyrunner::{
57	ChunkDelimiter,
58	PipeSz,
59	PyRunner,
60	PythonToUse,
61	RECV_TIMEOUT,
62	};
63
64	/// Minimum acceptable Python version, checked during venv creation
65	const PYTHON_VERSION_MIN: &str = "3.9";
66
67	/// pipe size for PyRunner instances used in venv creation
68	const PIPE_SZ: PipeSz = 16384;
69
70	/// chunk delimiter for PyRunner instances used during venv creation
71	const CHUNK_DELIMITER: ChunkDelimiter = b'\n';
72
73	/// only for user-facing help messages.
74	/// XXX: this must match the path used in `venv_path()`
75	pub const PYTHON_VENV_PATH_DEFAULT: &str = "~/.config/s4/venv";
76
77	/// Python project name
78	const PROJECT_NAME: &str = "s4_event_readers";
79
80	/// embedded files of the s4_event_readers Python project.
81	/// unpacked and installed during venv creation
82	static PY_PROJECT_DIR: Include_Dir = include_dir!("$CARGO_MANIFEST_DIR/src/python/s4_event_readers");
83
84	/// return path to python s4 venv directory.
85	/// does not check if it exists
86	pub fn venv_path() -> PathBuf {	4✔
87	#[allow(deprecated)]
88	let mut home: PathBuf = match home_dir() {	4✔
89	Some(h) => h,	4✔
90	None => {
91	// TODO: what is a better fallback path?
92	temp_dir()	×
93	},
94	};
95	home.push(".config");	4✔
96	home.push("s4");	4✔
97	home.push("venv");	4✔
98
99	if cfg!(test) {	4✔
100	// for tests, use a temporary path	4✔
101	home = temp_dir();	4✔
102	home.push("tmp-s4-test-python-venv");	4✔
103	}	4✔
104
105	defñ!("return {:?}", home);	4✔
106
107	home	4✔
108	}	4✔
109
110	/// copy the python project into a temporary directory
111	/// for build and installation
112	pub fn deploy_pyproject_s4_event_readers() -> Result<TempDir> {	2✔
113	defn!();	2✔
114
115	let tmpdir: TempDir = match TempDir::with_prefix(format!("{}_", PROJECT_NAME)) {	2✔
116	Ok(td) => td,	2✔
117	Err(err) => {	×
118	defx!("TempDir::new() error: {}", err);	×
119	return Result::Err(err);	×
120	}
121	};
122
123	defo!("PY_PROJECT_DIR.extract({:?})", tmpdir.path());	2✔
124	match PY_PROJECT_DIR.extract(tmpdir.path()) {	2✔
125	Ok(_) => {}	2✔
126	Err(err) => {	×
127	defx!("dir.extract error: {}", err);	×
128	return Result::Err(err);	×
129	}
130	}
131	defx!("Extracted PY_PROJECT_DIR {:?}", PY_PROJECT_DIR.path());	2✔
132
133	Result::Ok(tmpdir)	2✔
134	}	2✔
135
136	/// extract and compare the version
137	/// return `Ok` if version is acceptable
138	/// `data` is the output of `python --version`, e.g. `b'Python 3.9.7\n'`
139	pub(crate) fn extract_compare_version(data: &Bytes) -> Result<()> {	4✔
140	def1n!();	4✔
141	// create regex to extract version
142	let version_re: Regex = match Regex::new(r"^Python (\d+)\.(\d+)\.(\d+)") {	4✔
143	Ok(re) => re,	4✔
144	Err(err) => {	×
145	def1x!("Regex::new returned Err {:?}", err);	×
146	return Err(	×
147	Error::new(	×
148	ErrorKind::Other,	×
149	format!("failed to create python version regex; {}", err),	×
150	)	×
151	);	×
152	}
153	};
154	// regex capture the data
155	let captures = match version_re.captures(data) {	4✔
156	Some(captures) => captures,	3✔
157	None => {
158	def1x!("version_re.captures returned None");	1✔
159	return Err(	1✔
160	Error::new(	1✔
161	ErrorKind::Other,	1✔
162	format!("failed to capture python version from output {:?}", data),	1✔
163	)	1✔
164	);	1✔
165	}
166	};
167	// get the captured part as a String
168	let version_str: String = match std::str::from_utf8(&captures[0]) {	3✔
169	Ok(s) => {	3✔
170	def1o!("Converted version capture to str: {:?}", s);	3✔
171	// remove "Python " prefix
172	let mut s: String = s.to_string();	3✔
173	s = s.replace("Python ", "");	3✔
174	def1o!("Extracted version string: {:?}", s);	3✔
175
176	s	3✔
177	}
178	Err(err) => {	×
179	def1x!("from_utf8 returned Err {:?}", err);	×
180	return Err(	×
181	Error::new(	×
182	ErrorKind::Other,	×
183	format!("failed to convert python version capture to str; {}", err),	×
184	)	×
185	);	×
186	}
187	};
188	def1o!("Found Python version {}", version_str);	3✔
189	// compare the version strings with `compare_to()`
190	match compare_to(&version_str, PYTHON_VERSION_MIN, Cmp::Ge) {	3✔
191	Ok(cmp_result) => {	3✔
192	if cmp_result {	3✔
193	def1o!("Python version {} is acceptable", version_str);	2✔
194	} else {
195	def1x!("Python version too low; return Unsupported");	1✔
196	return Err(	1✔
197	Error::new(	1✔
198	ErrorKind::Unsupported,	1✔
199	format!("python version {} is less than the required minimum {}", version_str, PYTHON_VERSION_MIN),	1✔
200	)	1✔
201	);	1✔
202	}
203	}
204	Err(err) => {	×
205	def1x!("compare_to returned Err {:?}", err);	×
206	return Err(	×
207	Error::new(	×
208	ErrorKind::Other,	×
209	format!("failed to compare python versions {:?}", err),	×
210	)	×
211	);	×
212	}
213	}
214	def1x!("return Ok");	2✔
215
216	Ok(())	2✔
217	}	4✔
218
219	/// create the Python virtual environment using [`PyRunner`]s
220	pub fn create() -> Result3E<()> {	1✔
221	def1n!();	1✔
222
223	// run `python --version` to sanity check Python
224	// using found Python interpreter
225	// TODO: warn if version is less than required minimum
226	let mut pyrunner = match PyRunner::new(	1✔
227	PythonToUse::EnvPath,	1✔
228	PathId::default(),	1✔
229	PIPE_SZ,	1✔
230	RECV_TIMEOUT,	1✔
231	Some(CHUNK_DELIMITER),	1✔
232	None,	1✔
233	None,	1✔
234	vec![	1✔
235	"--version",	1✔
236	]) {	1✔
237	Ok(pyrunner) => pyrunner,	1✔
238	Err(err) => {	×
239	de_err!("Failed to create first Python runner: {}", err);	×
240	def1x!("Python --version; return Err {:?}", err);	×
241	return Result3E::Err(err);	×
242	}
243	};
244	match pyrunner.run(true, true, true) {	1✔
245	Ok((stdout, _stderr)) => {	1✔
246	match extract_compare_version(&stdout) {	1✔
247	Ok(_) => {},	1✔
248	Err(err) if err.kind() == ErrorKind::Unsupported => {	×
249	e_wrn!("{}", err.to_string());	×
250	}	×
251	Err(err) => {	×
252	e_err!("Failed to compare python version: {}", err);	×
253	def1x!("pyrunner.run() returned Err {:?}", err);	×
254	return Result3E::ErrNoReprint(err);	×
255	}
256	}
257	}
258	Err(err) => {	×
259	e_err!("Failed to run python --version: {}", err);	×
260	def1x!("pyrunner.run() returned Err {:?}", err);	×
261	return Result3E::ErrNoReprint(err);	×
262	}
263	}
264	// remember the python path used
265	let python_path = pyrunner.python_path.clone();	1✔
266
267	// rm the prior venv
268	let venv_path_pb: PathBuf = venv_path();	1✔
269	if venv_path_pb.exists() {	1✔
270	def1o!("remove_dir_all({:?})", venv_path_pb);	×
271	eprintln!("remove_dir_all({})\n", venv_path_pb.display());	×
272	match remove_dir_all(venv_path_pb.as_path()) {	×
273	Result::Ok(_) => {},	×
274	Result::Err(err) => {	×
275	e_err!("Failed to remove virtual environment directory {:?}: {}", venv_path_pb, err);	×
276	def1x!("remove_dir_all returned {:?}", err);	×
277	return Result3E::ErrNoReprint(err);	×
278	}
279	}
280	}	1✔
281
282	// create the venv directory including parent directories
283	// using found Python interpreter
284	def1o!("create_dir_all({:?})", venv_path_pb);	1✔
285	eprintln!("create_dir_all({})\n", venv_path_pb.display());	1✔
286	match create_dir_all(venv_path_pb.as_path()) {	1✔
287	Result::Ok(_) => {},	1✔
288	Result::Err(err) => {	×
289	e_err!("Failed to create virtual environment directory {:?}: {}", venv_path_pb, err);	×
290	def1x!("create_dir_all returned {:?}", err);	×
291	return Result3E::ErrNoReprint(err);	×
292	}
293	}
294
295	// one more sanity check
296	if ! venv_path_pb.is_dir() {	1✔
297	let err_msg = format!("Python virtual environment path {:?} is not a directory", venv_path_pb);	×
298	e_err!("{}", err_msg);	×
299	def1x!("{}", err_msg);	×
300	return Result3E::ErrNoReprint(Error::new(ErrorKind::NotADirectory, err_msg));	×
301	}	1✔
302
303	// create the venv using found Python interpreter
304	let venv_path_s: &str = match venv_path_pb.as_os_str().to_str() {	1✔
305	Some(s) => s,	1✔
306	None => {
307	def1x!("failed convert path to os_str to str {:?}, return Unsupported", venv_path_pb);	×
308	return Result3E::Err(	×
309	Error::new(	×
310	ErrorKind::Unsupported,	×
311	format!("failed to convert path to os_str to str; {:?}", venv_path_pb),	×
312	)	×
313	);	×
314	}
315	};
316	match PyRunner::run_once(	1✔
317	PythonToUse::Value,	1✔
318	PIPE_SZ,	1✔
319	RECV_TIMEOUT,	1✔
320	CHUNK_DELIMITER,	1✔
321	Some(python_path.to_string()),	1✔
322	vec![	1✔
323	"-m",	1✔
324	"venv",	1✔
325	"--clear",	1✔
326	"--prompt",	1✔
327	"s4",	1✔
328	venv_path_s,	1✔
329	],	1✔
330	true,	1✔
331	) {	1✔
332	Ok(_) => {},	1✔
333	Result::Err(err) => {	×
334	e_err!("Failed to create Python virtual environment; venv command failed: {}", err);	×
335	def1x!("pyrunner.run() returned {:?}", err);	×
336	return Result3E::ErrNoReprint(err);	×
337	}
338	}
339
340	// ensure pip is installed in the venv
341	match PyRunner::run_once(	1✔
342	PythonToUse::Venv,	1✔
343	PIPE_SZ,	1✔
344	RECV_TIMEOUT,	1✔
345	CHUNK_DELIMITER,	1✔
346	None,	1✔
347	vec![	1✔
348	"-m",	1✔
349	"ensurepip",	1✔
350	],	1✔
351	true,	1✔
352	) {	1✔
353	Ok(_) => {},	1✔
354	Err(err) => {	×
355	e_err!("Failed to ensurepip: {}", err);	×
356	def1x!("PyRunner::new failed {:?}", err);	×
357	return Result3E::ErrNoReprint(err);	×
358	}
359	};
360
361	// prevent pip from version checks
362	match PyRunner::run_once(	1✔
363	PythonToUse::Venv,	1✔
364	PIPE_SZ,	1✔
365	RECV_TIMEOUT,	1✔
366	CHUNK_DELIMITER,	1✔
367	None,	1✔
368	vec![	1✔
369	"-m",	1✔
370	"pip",	1✔
371	"config",	1✔
372	"set",	1✔
373	"--site",	1✔
374	"global.disable-pip-version-check",	1✔
375	"true",	1✔
376	],	1✔
377	true,	1✔
378	) {	1✔
379	Ok(_) => {},	1✔
380	Err(err) => {	×
381	e_err!("Failed to disable pip version check: {}", err);	×
382	def1x!("PyRunner::new failed {:?}", err);	×
383	return Result3E::ErrNoReprint(err);	×
384	}
385	};
386	match PyRunner::run_once(	1✔
387	PythonToUse::Venv,	1✔
388	PIPE_SZ,	1✔
389	RECV_TIMEOUT,	1✔
390	CHUNK_DELIMITER,	1✔
391	None,	1✔
392	vec![	1✔
393	"-m",	1✔
394	"pip",	1✔
395	"config",	1✔
396	"set",	1✔
397	"--site",	1✔
398	"global.disable-python-version-warning",	1✔
399	"true",	1✔
400	],	1✔
401	true,	1✔
402	) {	1✔
403	Ok(_) => {},	1✔
404	Err(err) => {	×
405	e_err!("Failed to disable python version warning: {}", err);	×
406	def1x!("PyRunner::new failed {:?}", err);	×
407	return Result3E::ErrNoReprint(err);	×
408	}
409	};
410
411	// expand the project into a temporary directory
412	let mut project_tmp_path: TempDir = match deploy_pyproject_s4_event_readers() {	1✔
413	Ok(p) => p,	1✔
414	Err(err) => {	×
415	e_err!("Failed to deploy python project: {}", err);	×
416	def1x!("deploy_pyproject_s4_event_readers failed {:?}", err);	×
417	return Result3E::ErrNoReprint(err);	×
418	}
419	};
420	if cfg!(debug_assertions) {	1✔
421	project_tmp_path.disable_cleanup(true);	1✔
422	def1o!("Temporary project remains at {:?}", project_tmp_path.path());	1✔
423	}	×
424	let project_tmp_path_s: &str = match project_tmp_path.path().as_os_str().to_str() {	1✔
425	Some(s) => s,	1✔
426	None => {
427	let err_msg = format!(	×
428	"failed to convert path to os_str to str; {:?}", project_tmp_path
429	);
430	e_err!("{}", err_msg);	×
431	def1x!("{}", err_msg);	×
432	return Result3E::Err(Error::other(err_msg));	×
433	}
434	};
435	eprintln!(	1✔
436	"inflated project {} to temporary path {:?}\n", PROJECT_NAME, project_tmp_path.path()	1✔
437	);
438
439	// install wheel
440	// this is purely to workaround using an older etl-parser package
441	// that uses legacy setup.py installation. without wheel
442	// the later project installation warns of a deprecated install method.
443	match PyRunner::run_once(	1✔
444	PythonToUse::Venv,	1✔
445	PIPE_SZ,	1✔
446	RECV_TIMEOUT,	1✔
447	CHUNK_DELIMITER,	1✔
448	None,	1✔
449	vec![	1✔
450	"-m",	1✔
451	"pip",	1✔
452	"install",	1✔
453	"wheel",	1✔
454	],	1✔
455	true,	1✔
456	) {	1✔
457	Ok(_) => {},	1✔
458	Err(err) => {	×
459	e_err!("Failed to ensurepip: {}", err);	×
460	def1x!("PyRunner::new failed {:?}", err);	×
461	return Result3E::ErrNoReprint(err);	×
462	}
463	};
464
465	// install required python packages
466	match PyRunner::run_once(	1✔
467	PythonToUse::Venv,	1✔
468	PIPE_SZ,	1✔
469	RECV_TIMEOUT,	1✔
470	CHUNK_DELIMITER,	1✔
471	None,	1✔
472	vec![	1✔
473	"-m",	1✔
474	"pip",	1✔
475	"install",	1✔
476	project_tmp_path_s,	1✔
477	],	1✔
478	true,	1✔
479	) {	1✔
480	Ok(_) => {},	1✔
481	Err(err) => {	×
482	e_err!("Failed to install python packages: {}", err);	×
483	def1x!("PyRunner::new failed {:?}", err);	×
484	return Result3E::ErrNoReprint(err);	×
485	}
486	};
487
488	// get site-packages path
489	let site_path: Bytes = match PyRunner::run_once(	1✔
490	PythonToUse::Venv,	1✔
491	PIPE_SZ,	1✔
492	RECV_TIMEOUT,	1✔
493	CHUNK_DELIMITER,	1✔
494	None,	1✔
495	vec![	1✔
496	"-c",	1✔
497	"import sysconfig; print(sysconfig.get_path(\"purelib\"))",	1✔
498	],	1✔
499	true,	1✔
500	) {	1✔
501	Ok((_, stdout, _)) => stdout,	1✔
502	Err(err) => {	×
503	e_wrn!("Failed to get site-packages path: {}", err);	×
504	debug_panic!("PyRunner::run_once failed {:?}", err);	×
505
506	Bytes::with_capacity(0)	×
507	}
508	};
509
510	let mut argv = vec![	1✔
511	"-m",
512	"compileall",	1✔
513	"-o0",	1✔
514	"-o1",	1✔
515	"-o2",	1✔
516	];
517	// add site-packages path if it was obtained
518	if ! site_path.is_empty() {	1✔
519	let site_path_s: &str = match std::str::from_utf8(&site_path) {	1✔
520	Ok(s) => s.trim(),	1✔
521	Err(err) => {	×
522	let err_msg = format!(	×
523	"failed to convert site-packages path to str; path {:?}, error {}",
524	site_path, err
525	);
526	e_err!("{}", err_msg);	×
527	debug_panic!("from_utf8 failed {:?}", err);	×
528	return Result3E::ErrNoReprint(Error::new(ErrorKind::Other, err_msg));	×
529	}
530	};
531	argv.push(site_path_s);	1✔
532	}	×
533
534	// precompile installed site-packages
535	match PyRunner::run_once(	1✔
536	PythonToUse::Venv,	1✔
537	PIPE_SZ,	1✔
538	RECV_TIMEOUT,	1✔
539	CHUNK_DELIMITER,	1✔
540	None,	1✔
541	argv,	1✔
542	true,	1✔
543	) {	1✔
544	Ok(_) => {},	1✔
545	Err(err) => {	×
546	e_wrn!("Failed to precompile python site-packages: {}; hopefully this can be ignored.", err);	×
547	debug_panic!("PyRunner::run_once failed {:?}", err);	×
548	}
549	};
550
551	if let Err(err) = PyRunner::run_once(	1✔
552	PythonToUse::Venv,	1✔
553	PIPE_SZ,	1✔
554	RECV_TIMEOUT,	1✔
555	CHUNK_DELIMITER,	1✔
556	None,	1✔
557	vec![	1✔
558	"-OO",	1✔
559	"-m",	1✔
560	"s4_event_readers",	1✔
561	],	1✔
562	true,	1✔
563	) {	1✔
564	e_err!("Failed to run s4_event_readers module test: {}", err);	×
565	def1x!("PyRunner::new failed {:?}", err);	×
566	return Result3E::ErrNoReprint(err);	×
567	}	1✔
568
569	// touch special flag file to mark the venv is fully created
570	let flag_path: PathBuf = venv_path().join("done");	1✔
571	if let Err(err) = std::fs::write(&flag_path, b"created by s4") {	1✔
572	e_err!("Failed to create {:?}: {}", flag_path, err);	×
573	def1x!("std::fs::write returned {:?}", err);	×
574	return Result3E::ErrNoReprint(err);	×
575	}	1✔
576
577	eprintln!("Python virtual environment created at {}", venv_path_pb.display());	1✔
578	eprintln!("This environment will be automatically used by s4 for Python-based event readers, i.e. for .asl, .etl, .odl files.");	1✔
579
580	def1x!("return Ok");	1✔
581
582	Result3E::Ok(())	1✔
583	}	1✔

jtmoon79 / super-speedy-syslog-searcher / 30326158672

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous