• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

pypest / pyemu / 5887625428

17 Aug 2023 06:23AM UTC coverage: 79.857% (+1.5%) from 78.319%
5887625428

push

github

briochh
Merge branch 'develop'

11386 of 14258 relevant lines covered (79.86%)

6.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

64.5
/pyemu/utils/os_utils.py
1
"""Operating system utilities in the PEST(++) realm
9✔
2
"""
3
import os
9✔
4
import sys
9✔
5
import platform
9✔
6
import shutil
9✔
7
import subprocess as sp
9✔
8
import multiprocessing as mp
9✔
9
import warnings
9✔
10
import socket
9✔
11
import time
9✔
12
from datetime import datetime
9✔
13
from ..pyemu_warnings import PyemuWarning
9✔
14

15
ext = ""
9✔
16
bin_path = os.path.join("..", "bin")
9✔
17
if "linux" in platform.platform().lower():
9✔
18
    bin_path = os.path.join(bin_path, "linux")
4✔
19
elif "darwin" in platform.platform().lower():
5✔
20
    bin_path = os.path.join(bin_path, "mac")
×
21
else:
22
    bin_path = os.path.join(bin_path, "win")
5✔
23
    ext = ".exe"
5✔
24

25
bin_path = os.path.abspath(bin_path)
9✔
26
os.environ["PATH"] += os.pathsep + bin_path
9✔
27

28

29
def _istextfile(filename, blocksize=512):
9✔
30
    """
31
    Function found from:
32
    https://eli.thegreenplace.net/2011/10/19/perls-guess-if-file-is-text-or-binary-implemented-in-python
33
    Returns True if file is most likely a text file
34
    Returns False if file is most likely a binary file
35
    Uses heuristics to guess whether the given file is text or binary,
36
    by reading a single block of bytes from the file.
37
    If more than 30% of the chars in the block are non-text, or there
38
    are NUL ('\x00') bytes in the block, assume this is a binary file.
39
    """
40

41
    import sys
×
42

43
    PY3 = sys.version_info[0] == 3
×
44

45
    # A function that takes an integer in the 8-bit range and returns
46
    # a single-character byte object in py3 / a single-character string
47
    # in py2.
48
    #
49
    int2byte = (lambda x: bytes((x,))) if PY3 else chr
×
50

51
    _text_characters = b"".join(int2byte(i) for i in range(32, 127)) + b"\n\r\t\f\b"
×
52
    block = open(filename, "rb").read(blocksize)
×
53
    if b"\x00" in block:
×
54
        # Files with null bytes are binary
55
        return False
×
56
    elif not block:
×
57
        # An empty file is considered a valid text file
58
        return True
×
59

60
    # Use translate's 'deletechars' argument to efficiently remove all
61
    # occurrences of _text_characters from the block
62
    nontext = block.translate(None, _text_characters)
×
63
    return float(len(nontext)) / len(block) <= 0.30
×
64

65

66
def _remove_readonly(func, path, excinfo):
9✔
67
    """remove readonly dirs, apparently only a windows issue
68
    add to all rmtree calls: shutil.rmtree(**,onerror=remove_readonly), wk"""
69
    os.chmod(path, 128)  # stat.S_IWRITE==128==normal
×
70
    func(path)
×
71

72

73
def run(cmd_str, cwd=".", verbose=False):
9✔
74
    """an OS agnostic function to execute a command line
75

76
    Args:
77
        cmd_str (`str`): the str to execute with `os.system()`
78

79
        cwd (`str`, optional): the directory to execute the command in.
80
            Default is ".".
81
        verbose (`bool`, optional): flag to echo to stdout the  `cmd_str`.
82
            Default is `False`.
83

84
    Notes:
85
        uses `platform` to detect OS and adds .exe suffix or ./ prefix as appropriate
86
        if `os.system` returns non-zero, an exception is raised
87

88
    Example::
89

90
        pyemu.os_utils.run("pestpp-ies my.pst",cwd="template")
91

92
    """
93
    bwd = os.getcwd()
9✔
94
    os.chdir(cwd)
9✔
95
    try:
9✔
96
        exe_name = cmd_str.split()[0]
9✔
97
        if "window" in platform.platform().lower():
9✔
98
            if not exe_name.lower().endswith("exe"):
4✔
99
                raw = cmd_str.split()
4✔
100
                raw[0] = exe_name + ".exe"
4✔
101
                cmd_str = " ".join(raw)
4✔
102
        else:
103
            if exe_name.lower().endswith("exe"):
5✔
104
                raw = cmd_str.split()
×
105
                exe_name = exe_name.replace(".exe", "")
×
106
                raw[0] = exe_name
×
107
                cmd_str = "{0} {1} ".format(*raw)
×
108
            if os.path.exists(exe_name) and not exe_name.startswith("./"):
5✔
109
                cmd_str = "./" + cmd_str
×
110

111
    except Exception as e:
×
112
        os.chdir(bwd)
×
113
        raise Exception("run() error preprocessing command line :{0}".format(str(e)))
×
114
    if verbose:
9✔
115
        print("run():{0}".format(cmd_str))
×
116

117
    try:
9✔
118
        ret_val = os.system(cmd_str)
9✔
119
    except Exception as e:
×
120
        os.chdir(bwd)
×
121
        raise Exception("run() raised :{0}".format(str(e)))
×
122
    os.chdir(bwd)
9✔
123

124
    if "window" in platform.platform().lower():
9✔
125
        if ret_val != 0:
4✔
126
            raise Exception("run() returned non-zero: {0}".format(ret_val))
4✔
127
    else:
128
        estat = os.WEXITSTATUS(ret_val)
5✔
129
        if estat != 0:
5✔
130
            raise Exception("run() returned non-zero: {0}".format(estat))
5✔
131

132

133
def _try_remove_existing(d, forgive=False):
9✔
134
    try:
9✔
135
        shutil.rmtree(d, onerror=_remove_readonly)  # , onerror=del_rw)
9✔
136
        return True
9✔
137
    except Exception as e:
×
138
        if not forgive:
×
139
            raise Exception(
×
140
                f"unable to remove existing dir: {d}\n{e}"
141
            )
142
        else:
143
            warnings.warn(
×
144
                f"unable to remove worker dir: {d}\n{e}",
145
                PyemuWarning,
146
            )
147
        return False
×
148

149

150
def _try_copy_dir(o_d, n_d):
9✔
151
    try:
9✔
152
        shutil.copytree(o_d, n_d)
9✔
153
    except PermissionError:
×
154
        time.sleep(3) # pause for windows locking issues
×
155
        try:
×
156
            shutil.copytree(o_d, n_d)
×
157
        except Exception as e:
×
158
            raise Exception(
×
159
                f"unable to copy files from base dir: "
160
                f"{o_d}, to new dir: {n_d}\n{e}"
161
            )
162

163

164
def start_workers(
9✔
165
    worker_dir,
166
    exe_rel_path,
167
    pst_rel_path,
168
    num_workers=None,
169
    worker_root="..",
170
    port=4004,
171
    rel_path=None,
172
    local=True,
173
    cleanup=True,
174
    master_dir=None,
175
    verbose=False,
176
    silent_master=False,
177
    reuse_master=False,
178
    restart=False
179
):
180
    """start a group of pest(++) workers on the local machine
181

182
    Args:
183
        worker_dir (`str`): the path to a complete set of input files need by PEST(++).
184
            This directory will be copied to make worker (and optionally the master)
185
            directories
186
        exe_rel_path (`str`): the relative path to and name of the pest(++) executable from within
187
            the `worker_dir`.  For example, if the executable is up one directory from
188
            `worker_dir`, the `exe_rel_path` would be `os.path.join("..","pestpp-ies")`
189
        pst_rel_path (`str`): the relative path to and name of the pest control file from within
190
            `worker_dir`.
191
        num_workers (`int`, optional): number of workers to start. defaults to number of cores
192
        worker_root (`str`, optional):  the root directory to make the new worker directories in.
193
            Default is ".."  (up one directory from where python is running).
194
        rel_path (`str`, optional): the relative path to where pest(++) should be run
195
            from within the worker_dir, defaults to the uppermost level of the worker dir.
196
            This option is usually not needed unless you are one of those crazy people who
197
            spreads files across countless subdirectories.
198
        local (`bool`, optional): flag for using "localhost" instead of actual hostname/IP address on
199
            worker command line. Default is True.  `local` can also be passed as an `str`, in which
200
            case `local` is used as the hostname (for example `local="192.168.10.1"`)
201
        cleanup (`bool`, optional):  flag to remove worker directories once processes exit. Default is
202
            True.  Set to False for debugging issues
203
        master_dir (`str`): name of directory for master instance.  If `master_dir`
204
            exists, then it will be REMOVED!!!  If `master_dir`, is None,
205
            no master instance will be started.  If not None, a copy of `worker_dir` will be
206
            made into `master_dir` and the PEST(++) executable will be started in master mode
207
            in this directory. Default is None
208
        verbose (`bool`, optional): flag to echo useful information to stdout.  Default is False
209
        silent_master (`bool`, optional): flag to pipe master output to devnull and instead print
210
            a simple message to stdout every few seconds.  This is only for
211
            pestpp Travis testing so that log file sizes dont explode. Default is False
212
        reuse_master (`bool`): flag to use an existing `master_dir` as is - this is an advanced user
213
            option for cases where you want to construct your own `master_dir` then have an async
214
            process started in it by this function.
215
        restart (`bool`): flag to add a restart flag to the master start. If `True`, this will include
216
            `/r` in the master call string.
217

218
    Notes:
219
        If all workers (and optionally master) exit gracefully, then the worker
220
        dirs will be removed unless `cleanup` is False
221

222
    Example::
223

224
        # start 10 workers using the directory "template" as the base case and
225
        # also start a master instance in a directory "master".
226
        pyemu.helpers.start_workers("template","pestpp-ies","pest.pst",10,master_dir="master",
227
                                    worker_root=".")
228

229
    """
230

231
    if not os.path.isdir(worker_dir):
9✔
232
        raise Exception("worker dir '{0}' not found".format(worker_dir))
×
233
    if not os.path.isdir(worker_root):
9✔
234
        raise Exception("worker root dir not found")
×
235
    if num_workers is None:
9✔
236
        num_workers = mp.cpu_count()
×
237
    else:
238
        num_workers = int(num_workers)
9✔
239
    # assert os.path.exists(os.path.join(worker_dir,rel_path,exe_rel_path))
240
    exe_verf = True
9✔
241

242
    if rel_path:
9✔
243
        if not os.path.exists(os.path.join(worker_dir, rel_path, exe_rel_path)):
×
244
            # print("warning: exe_rel_path not verified...hopefully exe is in the PATH var")
245
            exe_verf = False
×
246
    else:
247
        if not os.path.exists(os.path.join(worker_dir, exe_rel_path)):
9✔
248
            # print("warning: exe_rel_path not verified...hopefully exe is in the PATH var")
249
            exe_verf = False
9✔
250
    if rel_path is not None:
9✔
251
        if not os.path.exists(os.path.join(worker_dir, rel_path, pst_rel_path)):
×
252
            raise Exception("pst_rel_path not found from worker_dir using rel_path")
×
253
    else:
254
        if not os.path.exists(os.path.join(worker_dir, pst_rel_path)):
9✔
255
            raise Exception("pst_rel_path not found from worker_dir")
×
256
    if isinstance(local, str):
9✔
257
        hostname = local
×
258
    elif local:
9✔
259
        hostname = "localhost"
9✔
260
    else:
261
        hostname = socket.gethostname()
×
262

263
    base_dir = os.getcwd()
9✔
264
    port = int(port)
9✔
265

266
    if os.path.exists(os.path.join(worker_dir, exe_rel_path)):
9✔
267
        if "window" in platform.platform().lower():
×
268
            if not exe_rel_path.lower().endswith("exe"):
×
269
                exe_rel_path = exe_rel_path + ".exe"
×
270
        else:
271
            if not exe_rel_path.startswith("./"):
×
272
                exe_rel_path = "./" + exe_rel_path
×
273

274
    if master_dir is not None:
9✔
275
        if master_dir != "." and os.path.exists(master_dir) and not reuse_master:
9✔
276
            _try_remove_existing(master_dir)
×
277
        if master_dir != "." and not reuse_master:
9✔
278
            _try_copy_dir(worker_dir, master_dir)
9✔
279
        
280
        args = [exe_rel_path, pst_rel_path, "/h", ":{0}".format(port)]
9✔
281
        if restart is True:
9✔
282
            # add restart if requested
283
            args = [exe_rel_path, pst_rel_path, "/h", "/r", ":{0}".format(port)]
×
284
        
285
        if rel_path is not None:
9✔
286
            cwd = os.path.join(master_dir, rel_path)
×
287
        else:
288
            cwd = master_dir
9✔
289
        if verbose:
9✔
290
            print("master:{0} in {1}".format(" ".join(args), cwd))
×
291
        stdout = None
9✔
292
        if silent_master:
9✔
293
            stdout = open(os.devnull, "w")
×
294
        try:
9✔
295
            os.chdir(cwd)
9✔
296
            master_p = sp.Popen(args, stdout=stdout)  # ,stdout=sp.PIPE,stderr=sp.PIPE)
9✔
297
            os.chdir(base_dir)
9✔
298
        except Exception as e:
×
299
            raise Exception("error starting master instance: {0}".format(str(e)))
×
300
        time.sleep(1.5)  # a few cycles to let the master get ready
9✔
301

302
    tcp_arg = "{0}:{1}".format(hostname, port)
9✔
303
    procs = []
9✔
304
    worker_dirs = []
9✔
305
    for i in range(num_workers):
9✔
306
        new_worker_dir = os.path.join(worker_root, "worker_{0}".format(i))
9✔
307
        if os.path.exists(new_worker_dir):
9✔
308
            _try_remove_existing(new_worker_dir)
×
309
        _try_copy_dir(worker_dir, new_worker_dir)
9✔
310
        try:
9✔
311
            if exe_verf:
9✔
312
                # if rel_path is not None:
313
                #     exe_path = os.path.join(rel_path,exe_rel_path)
314
                # else:
315
                exe_path = exe_rel_path
×
316
            else:
317
                exe_path = exe_rel_path
9✔
318
            args = [exe_path, pst_rel_path, "/h", tcp_arg]
9✔
319
            # print("starting worker in {0} with args: {1}".format(new_worker_dir,args))
320
            if rel_path is not None:
9✔
321
                cwd = os.path.join(new_worker_dir, rel_path)
×
322
            else:
323
                cwd = new_worker_dir
9✔
324

325
            os.chdir(cwd)
9✔
326
            if verbose:
9✔
327
                print("worker:{0} in {1}".format(" ".join(args), cwd))
×
328
            with open(os.devnull, "w") as f:
9✔
329
                p = sp.Popen(args, stdout=f, stderr=f)
9✔
330
            procs.append(p)
9✔
331
            os.chdir(base_dir)
9✔
332
        except Exception as e:
×
333
            raise Exception("error starting worker: {0}".format(str(e)))
×
334
        worker_dirs.append(new_worker_dir)
9✔
335

336
    if master_dir is not None:
9✔
337
        # while True:
338
        #     line = master_p.stdout.readline()
339
        #     if line != '':
340
        #         print(str(line.strip())+'\r',end='')
341
        #     if master_p.poll() is not None:
342
        #         print(master_p.stdout.readlines())
343
        #         break
344
        if silent_master:
9✔
345
            # this keeps travis from thinking something is wrong...
346
            while True:
347
                rv = master_p.poll()
×
348
                if master_p.poll() is not None:
×
349
                    break
×
350
                print(datetime.now(), "still running")
×
351
                time.sleep(5)
×
352
        else:
353
            master_p.wait()
9✔
354
            time.sleep(1.5)  # a few cycles to let the workers end gracefully
9✔
355

356
        # kill any remaining workers
357
        for p in procs:
9✔
358
            p.kill()
9✔
359
    # this waits for sweep to finish, but pre/post/model (sub)subprocs may take longer
360
    for p in procs:
9✔
361
        p.wait()
9✔
362
    if cleanup:
9✔
363
        cleanit = 0
9✔
364
        removed = set()
9✔
365
        while len(removed) < len(worker_dirs):  # arbitrary 100000 limit
9✔
366
            cleanit = cleanit + 1
9✔
367
            for d in worker_dirs:
9✔
368
                if os.path.exists(d):
9✔
369
                    success = _try_remove_existing(d, forgive=True)
9✔
370
                    if success:
9✔
371
                        removed.update(d)
9✔
372
                else:
373
                    removed.update(d)
1✔
374
            if cleanit > 100:
9✔
375
                break
1✔
376

377

378
    if master_dir is not None:
9✔
379
        ret_val = master_p.returncode
9✔
380
        if ret_val != 0:
9✔
381
            raise Exception("start_workers() master returned non-zero: {0}".format(ret_val))
×
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc