• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

pypest / pyemu / 5887625428

17 Aug 2023 06:23AM UTC coverage: 79.857% (+1.5%) from 78.319%
5887625428

push

github

briochh
Merge branch 'develop'

11386 of 14258 relevant lines covered (79.86%)

6.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

82.86
/pyemu/eds.py
1
from __future__ import print_function, division
9✔
2
import os
9✔
3
import copy
9✔
4
import shutil
9✔
5
from datetime import datetime
9✔
6
import warnings
9✔
7
from .pyemu_warnings import PyemuWarning
9✔
8
import numpy as np
9✔
9
import pandas as pd
9✔
10
from pyemu.en import ObservationEnsemble
9✔
11
from pyemu.mat.mat_handler import Matrix, Jco, Cov
9✔
12
from pyemu.pst.pst_handler import Pst
9✔
13
from pyemu.utils.os_utils import _istextfile,run
9✔
14
from .logger import Logger
9✔
15

16

17

18
class EnDS(object):
9✔
19
    """Ensemble Data Space Analysis using the approach of He et al (2018) (https://doi.org/10.2118/182609-PA)
9✔
20

21
    Args:
22
        pst (varies): something that can be cast into a `pyemu.Pst`.  Can be an `str` for a
23
            filename or an existing `pyemu.Pst`.
24
        sim_ensemble (varies): something that can be cast into a `pyemu.ObservationEnsemble`.  Can be
25
            an `str` for a  filename or `pd.DataFrame` or an existing `pyemu.ObservationEnsemble`.
26
        noise_ensemble (varies): something that can be cast into a `pyemu.ObservationEnsemble` that is the
27
            obs+noise realizations.  If not passed, a noise ensemble is generated using either `obs_cov` or the
28
            information in `pst` (i.e. weights or standard deviations)
29
        obscov (varies, optional): observation noise covariance matrix.  If `str`, a filename is assumed and
30
            the noise covariance matrix is loaded from a file using
31
            the file extension (".jcb"/".jco" for binary, ".cov"/".mat" for PEST-style ASCII matrix,
32
            or ".unc" for uncertainty files).  If `None`, the noise covariance matrix is
33
            constructed from the observation weights (and optionally "standard_deviation")
34
            .  Can also be a `pyemu.Cov` instance
35
        forecasts (enumerable of `str`): the names of the entries in `pst.osbervation_data` (and in `ensemble`).
36
        verbose (`bool`): controls screen output.  If `str`, a filename is assumed and
37
                and log file is written.
38
        
39

40
    Example::
41

42
        #assumes "my.pst" exists
43
        ends = pyemu.EnDS(ensemble="my.0.obs.jcb",forecasts=["fore1","fore2"])
44
        ends.get_posterior_prediction_moments() #similar to Schur-style data worth
45
        ends.prep_for_dsi() #setup a new pest interface() based on the DSI approach
46

47

48
    """
49

50
    def __init__(
9✔
51
        self,
52
        pst=None,
53
        sim_ensemble=None,
54
        noise_ensemble=None,
55
        obscov=None,
56
        predictions=None,
57
        verbose=False,
58
    ):
59
        self.logger = Logger(verbose)
8✔
60
        self.log = self.logger.log
8✔
61
        self.sim_en_arg = sim_ensemble
8✔
62
        # if jco is None:
63
        self.__sim_en = sim_ensemble
8✔
64

65
        self.pst_arg = pst
8✔
66
        if obscov is None and pst is not None:
8✔
67
            obscov = pst
8✔
68

69
        # private attributes - access is through @decorated functions
70
        self.__pst = None
8✔
71
        self.__obscov = None
8✔
72
        self.__sim_ensemble = None
8✔
73
        self.__noise_ensemble = None
8✔
74

75

76
        self.log("pre-loading base components")
8✔
77
        self.sim_ensemble_arg = sim_ensemble
8✔
78
        if sim_ensemble is not None:
8✔
79
            self.__sim_ensemble = self.__load_ensemble(self.sim_ensemble_arg)
8✔
80
        if self.sim_ensemble is None:
8✔
81
            raise Exception("sim_ensemble is required for EnDS")
×
82
        self.noise_ensemble_arg = noise_ensemble
8✔
83
        if noise_ensemble is not None:
8✔
84
            self.__noise_ensemble = self.__load_ensemble(self.noise_ensemble_arg)
×
85
        if pst is not None:
8✔
86
            self.__load_pst()
8✔
87
        if pst is None:
8✔
88
            raise Exception("pst is required for EnDS")
×
89
        self.obscov_arg = obscov
8✔
90
        if obscov is not None:
8✔
91
            self.__load_obscov()
8✔
92

93
        self.predictions = predictions
8✔
94
        if predictions is None and self.pst is not None:
8✔
95
            if self.pst.forecast_names is not None:
8✔
96
                self.predictions = self.pst.forecast_names
8✔
97
        if self.predictions is None:
8✔
98
            raise Exception("predictions are required for EnDS")
×
99
        if isinstance(self.predictions,list):
8✔
100
            self.predictions = [p.strip().lower() for p in self.predictions]
8✔
101

102
        self.log("pre-loading base components")
8✔
103

104
        # automatically do some things that should be done
105
        assert type(self.obscov) == Cov
8✔
106

107
    def __fromfile(self, filename, astype=None):
9✔
108
        """a private method to deduce and load a filename into a matrix object.
109
        Uses extension: 'jco' or 'jcb': binary, 'mat','vec' or 'cov': ASCII,
110
        'unc': pest uncertainty file.
111

112
        """
113
        assert os.path.exists(filename), (
8✔
114
            "LinearAnalysis.__fromfile(): " + "file not found:" + filename
115
        )
116
        ext = filename.split(".")[-1].lower()
8✔
117
        if ext in ["jco", "jcb"]:
8✔
118
            self.log("loading jcb format: " + filename)
×
119
            if astype is None:
×
120
                astype = Jco
×
121
            m = astype.from_binary(filename)
×
122
            self.log("loading jcb format: " + filename)
×
123
        elif ext in ["mat", "vec"]:
8✔
124
            self.log("loading ascii format: " + filename)
8✔
125
            if astype is None:
8✔
126
                astype = Matrix
×
127
            m = astype.from_ascii(filename)
8✔
128
            self.log("loading ascii format: " + filename)
8✔
129
        elif ext in ["cov"]:
8✔
130
            self.log("loading cov format: " + filename)
×
131
            if astype is None:
×
132
                astype = Cov
×
133
            if _istextfile(filename):
×
134
                m = astype.from_ascii(filename)
×
135
            else:
136
                m = astype.from_binary(filename)
×
137
            self.log("loading cov format: " + filename)
×
138
        elif ext in ["unc"]:
8✔
139
            self.log("loading unc file format: " + filename)
8✔
140
            if astype is None:
8✔
141
                astype = Cov
×
142
            m = astype.from_uncfile(filename)
8✔
143
            self.log("loading unc file format: " + filename)
8✔
144
        elif ext in ["csv"]:
8✔
145
            self.log("loading csv format: " + filename)
8✔
146
            if astype is None:
8✔
147
                astype = ObservationEnsemble
×
148
            m = astype.from_csv(self.pst,filename=filename)
8✔
149
            self.log("loading csv format: " + filename)
8✔
150
        else:
151
            raise Exception(
×
152
                "EnDS.__fromfile(): unrecognized"
153
                + " filename extension:"
154
                + str(ext)
155
            )
156
        return m
8✔
157

158
    def __load_pst(self):
9✔
159
        """private method set the pst attribute"""
160
        if self.pst_arg is None:
8✔
161
            return None
×
162
        if isinstance(self.pst_arg, Pst):
8✔
163
            self.__pst = self.pst_arg
8✔
164
            return self.pst
8✔
165
        else:
166
            try:
8✔
167
                self.log("loading pst: " + str(self.pst_arg))
8✔
168
                self.__pst = Pst(self.pst_arg)
8✔
169
                self.log("loading pst: " + str(self.pst_arg))
8✔
170
                return self.pst
8✔
171
            except Exception as e:
×
172
                raise Exception(
×
173
                    "EnDS.__load_pst(): error loading"
174
                    + " pest control from argument: "
175
                    + str(self.pst_arg)
176
                    + "\n->"
177
                    + str(e)
178
                )
179

180
    def __load_ensemble(self,arg):
9✔
181
        """private method to set the ensemble attribute from a file or a dataframe object"""
182
        if self.sim_ensemble_arg is None:
8✔
183
            return None
×
184
            # raise Exception("linear_analysis.__load_jco(): jco_arg is None")
185
        if isinstance(arg, Matrix):
8✔
186
            ensemble = ObservationEnsemble(pst=self.pst, df=arg.to_dataframe())
×
187
        elif isinstance(self.sim_ensemble_arg, str):
8✔
188
            ensemble = self.__fromfile(arg, astype=ObservationEnsemble)
8✔
189
        elif isinstance(arg, ObservationEnsemble):
8✔
190
            ensemble = arg.copy()
8✔
191
        else:
192
            raise Exception(
×
193
                "EnDS.__load_ensemble(): arg must "
194
                + "be a matrix object, dataframe, or a file name: "
195
                + str(arg)
196
            )
197
        return ensemble
8✔
198

199

200

201
    def __load_obscov(self):
9✔
202
        """private method to set the obscov attribute from:
203
        a pest control file (observation weights)
204
        a pst object
205
        a matrix object
206
        an uncert file
207
        an ascii matrix file
208
        """
209
        # if the obscov arg is None, but the pst arg is not None,
210
        # reset and load from obs weights
211
        self.log("loading obscov")
8✔
212
        if not self.obscov_arg:
8✔
213
            if self.pst_arg:
×
214
                self.obscov_arg = self.pst_arg
×
215
            else:
216
                raise Exception(
×
217
                    "linear_analysis.__load_obscov(): " + "obscov_arg is None"
218
                )
219
        if isinstance(self.obscov_arg, Matrix):
8✔
220
            self.__obscov = self.obscov_arg
8✔
221
            return
8✔
222

223
        elif isinstance(self.obscov_arg, str):
8✔
224
            if self.obscov_arg.lower().endswith(".pst"):
8✔
225
                self.__obscov = Cov.from_obsweights(self.obscov_arg)
8✔
226
            else:
227
                self.__obscov = self.__fromfile(self.obscov_arg, astype=Cov)
8✔
228
        elif isinstance(self.obscov_arg, Pst):
8✔
229
            self.__obscov = Cov.from_observation_data(self.obscov_arg)
8✔
230
        else:
231
            raise Exception(
×
232
                "EnDS.__load_obscov(): "
233
                + "obscov_arg must be a "
234
                + "matrix object or a file name: "
235
                + str(self.obscov_arg)
236
            )
237
        self.log("loading obscov")
8✔
238

239

240

241
    # these property decorators help keep from loading potentially
242
    # unneeded items until they are called
243
    # returns a reference - cheap, but can be dangerous
244

245
    @property
9✔
246
    def sim_ensemble(self):
9✔
247
        return self.__sim_en
8✔
248

249
    @property
9✔
250
    def obscov(self):
9✔
251
        """get the observation noise covariance matrix attribute
252

253
        Returns:
254
            `pyemu.Cov`: a reference to the `LinearAnalysis.obscov` attribute
255

256
        """
257
        if not self.__obscov:
8✔
258
            self.__load_obscov()
×
259
        return self.__obscov
8✔
260

261

262
    @property
9✔
263
    def pst(self):
9✔
264
        """the pst attribute
265

266
        Returns:
267
            `pyemu.Pst`: the pst attribute
268

269
        """
270
        if self.__pst is None and self.pst_arg is None:
8✔
271
            raise Exception(
×
272
                "linear_analysis.pst: can't access self.pst:"
273
                + "no pest control argument passed"
274
            )
275
        elif self.__pst:
8✔
276
            return self.__pst
8✔
277
        else:
278
            self.__load_pst()
8✔
279
            return self.__pst
8✔
280

281
    def reset_pst(self, arg):
9✔
282
        """reset the EnDS.pst attribute
283

284
        Args:
285
            arg (`str` or `pyemu.Pst`): the value to assign to the pst attribute
286

287
        """
288
        self.logger.statement("resetting pst")
×
289
        self.__pst = None
×
290
        self.pst_arg = arg
×
291

292
    def reset_obscov(self, arg=None):
9✔
293
        """reset the obscov attribute to None
294

295
        Args:
296
            arg (`str` or `pyemu.Matrix`): the value to assign to the obscov
297
                attribute.  If None, the private __obscov attribute is cleared
298
                but not reset
299
        """
300
        self.logger.statement("resetting obscov")
×
301
        self.__obscov = None
×
302
        if arg is not None:
×
303
            self.obscov_arg = arg
×
304

305
    def get_posterior_prediction_convergence_summary(self,num_realization_sequence,num_replicate_sequence,
9✔
306
                                             obslist_dict=None):
307
        """repeatedly run `EnDS.get_predictive_posterior_moments() with less than all the possible
308
        realizations to evaluate whether the uncertainty estimates have converged
309

310
        Args:
311
            num_realization_sequence (`[int']): the sequence of realizations to test.
312
            num_replicate_sequence (`[int]`): The number of replicates of randomly selected realizations to test
313
                for each `num_realization_sequence` value.  For example, if num_realization_sequence is [10,100,1000]
314
                and num_replicated_sequence is [4,5,6], then `EnDS.get_predictive posterior_moments()` is called 4
315
                times using 10 randomly selected realizations (new realizations selected 4 times), 5 times using
316
                100 randomly selected realizations, and then 6 times using 1000 randomly selected realizations.
317
            obslist_dict (`dict`, optional): a nested dictionary-list of groups of observations
318
                to pass to `EnDS.get_predictive_posterior_moments()`.
319

320
        Returns:
321
             `dict`: a dictionary of num_reals: `pd.DataFrame` pairs, where the dataframe is the mean
322
                predictive standard deviation results from calling `EnDS.get_predictive_posterior_moments()` for the
323
                desired number of replicates.
324

325
         Example::
326

327
            ends = pyemu.EnDS(pst="my.pst",sim_ensemble="my.0.obs.csv",predictions=["predhead","predflux"])
328
            obslist_dict = {"hds":["head1","head2"],"flux":["flux1","flux2"]}
329
            num_reals_seq = [10,20,30,100,1000] # assuming there are 1000 reals in "my.0.obs.csv"]
330
            num_reps_seq = [5,5,5,5,5]
331
            mean_dfs = sc.get_posterior_prediction_convergence_summary(num_reals_seq,num_reps_seq,
332
                obslist_dict=obslist_dict)
333

334
        """
335
        real_idx = np.arange(self.sim_ensemble.shape[0],dtype=int)
×
336
        results = {}
×
337
        for nreals,nreps in zip(num_realization_sequence,num_replicate_sequence):
×
338
            rep_results = []
×
339
            print("-->testing ",nreals)
×
340
            for rep in range(nreps):
×
341
                rreals = np.random.choice(real_idx,nreals,False)
×
342
                sim_ensemble = self.sim_ensemble.iloc[rreals,:].copy()
×
343
                _,dfstd,_ = self.get_posterior_prediction_moments(obslist_dict=obslist_dict,
×
344
                                                                 sim_ensemble=sim_ensemble,
345
                                                                 include_first_moment=False)
346
                rep_results.append(dfstd)
×
347
            results[nreals] = rep_results
×
348

349
        means = {}
×
350
        for nreals,dfs in results.items():
×
351
            mn = dfs[0]
×
352
            for df in dfs[1:]:
×
353
                mn += df
×
354
            mn /= len(dfs)
×
355
            means[nreals] = mn
×
356

357
        return means
×
358

359

360
    def get_posterior_prediction_moments(self, obslist_dict=None,sim_ensemble=None,include_first_moment=True):
9✔
361
        """A dataworth method to analyze the posterior (expected) mean and uncertainty as a result of conditioning with
362
         some additional observations not used in the conditioning of the current ensemble results.
363

364
        Args:
365
            obslist_dict (`dict`, optional): a nested dictionary-list of groups of observations
366
                that are to be treated as gained/collected.  key values become
367
                row labels in returned dataframe. If `None`, then every zero-weighted
368
                observation is tested sequentially. Default is `None`
369
            sim_ensemble (`pyemu.ObservationEnsemble`): the simulation results ensemble to use.
370
                If `None`, `self.sim_ensemble` is used.  Default is `None`
371

372
            include_first_moment (`bool`): flag to include calculations of the predictive first moments.
373
                This can slow things down,so if not needed, better to skip.  Default is `True`
374

375
        Returns:
376
            tuple containing
377

378
            - **dict**: dictionary of first-moment dataframes. Keys are `obslist_dict` keys.  If `include_first_moment`
379
                is None, this is an empty dict.
380
            - **pd.DataFrame**: prediction standard deviation summary
381
            - **pd.DataFrame**: precent prediction standard deviation reduction summary
382

383

384
        Example::
385

386
            ends = pyemu.EnDS(pst="my.pst",sim_ensemble="my.0.obs.csv",predictions=["predhead","predflux"])
387
            obslist_dict = {"hds":["head1","head2"],"flux":["flux1","flux2"]}
388
            mean_dfs,dfstd,dfpercent = sc.get_posterior_prediction_moments(obslist_dict=obslist_dict)
389

390
        """
391

392

393
        if obslist_dict is not None:
8✔
394
            if type(obslist_dict) == list:
8✔
395
                obslist_dict = dict(zip(obslist_dict, obslist_dict))
×
396
            obs = self.pst.observation_data
8✔
397
            onames = []
8✔
398
            [onames.extend(names) for gp,names in obslist_dict.items()]
8✔
399
            oobs = obs.loc[onames,:]
8✔
400
            zobs = oobs.loc[oobs.weight==0,"obsnme"].tolist()
8✔
401

402
            if len(zobs) > 0:
8✔
403
                raise Exception(
×
404
                    "Observations in obslist_dict must have "
405
                    + "nonzero weight. The following observations "
406
                    + "violate that condition: "
407
                    + ",".join(zobs)
408
                )
409
        else:
410
            obslist_dict = dict(zip(self.pst.nnz_obs_names, self.pst.nnz_obs_names))
×
411
            onames = self.pst.nnz_obs_names
×
412

413
        if "posterior" not in obslist_dict:
8✔
414
            obslist_dict["posterior"] = onames
8✔
415

416
        names = onames
8✔
417
        names.extend(self.predictions)
8✔
418
        self.logger.log("getting deviations")
8✔
419
        if sim_ensemble is None:
8✔
420
            sim_ensemble = self.sim_ensemble
8✔
421
        oe = sim_ensemble.loc[:,names].get_deviations() / np.sqrt(float(sim_ensemble.shape[0] - 1))
8✔
422
        self.logger.log("getting deviations")
8✔
423
        self.logger.log("forming cov matrix")
8✔
424
        cmat = (np.dot(oe.values.transpose(),oe.values))
8✔
425
        cdf = pd.DataFrame(cmat, index=names, columns=names)
8✔
426
        self.logger.log("forming cov matrix")
8✔
427

428
        var_data = {}
8✔
429
        prior_var = sim_ensemble.loc[:, self.predictions].std() ** 2
8✔
430
        prior_mean = sim_ensemble.loc[:, self.predictions].mean()
8✔
431
        var_data["prior"] = prior_var
8✔
432
        groups = list(obslist_dict.keys())
8✔
433
        groups.sort()
8✔
434

435
        mean_dfs = {}
8✔
436
        for group in groups:
8✔
437
            self.logger.log("processing "+group)
8✔
438
            onames = obslist_dict[group]
8✔
439
            self.logger.log("extracting blocks from cov matrix")
8✔
440

441
            dd = cdf.loc[onames,onames].values.copy()
8✔
442
            ccov = cdf.loc[onames,self.predictions].values.copy()
8✔
443
            self.logger.log("extracting blocks from cov matrix")
8✔
444

445
            self.logger.log("adding noise cov to data block")
8✔
446
            dd += self.obscov.get(onames,onames).x
8✔
447
            self.logger.log("adding noise cov to data block")
8✔
448

449
            #todo: test for inveribility and shrink if needed...
450
            self.logger.log("inverting data cov block")
8✔
451
            dd = np.linalg.inv(dd)
8✔
452
            self.logger.log("inverting data cov block")
8✔
453
            pt_var = []
8✔
454
            pt_mean = {}
8✔
455

456

457
            if include_first_moment:
8✔
458
                self.logger.log("preping first moment pieces")
8✔
459
                omean = sim_ensemble.loc[:, onames].mean().values
8✔
460
                reals = sim_ensemble.index.values
8✔
461
                innovation_vecs = {real: sim_ensemble.loc[real, onames].values - omean for real in sim_ensemble.index}
8✔
462
                self.logger.log("preping first moment pieces")
8✔
463

464
            for i,p in enumerate(self.predictions):
8✔
465
                self.logger.log("calc second moment for "+p)
8✔
466
                ccov_vec = ccov[:,i]
8✔
467
                first_term = np.dot(ccov_vec.transpose(), dd)
8✔
468
                schur = np.dot(first_term,ccov_vec)
8✔
469
                post_var = prior_var[i] - schur
8✔
470
                pt_var.append(post_var)
8✔
471
                self.logger.log("calc second moment for " + p)
8✔
472
                if include_first_moment:
8✔
473
                    self.logger.log("calc first moment values for "+p)
8✔
474
                    mean_vals = []
8✔
475
                    prmn = prior_mean[p]
8✔
476
                    for real in reals:
8✔
477
                        mn = prmn + np.dot(first_term,innovation_vecs[real])
8✔
478
                        mean_vals.append(mn)
8✔
479
                    pt_mean[p] = np.array(mean_vals)
8✔
480
                    self.logger.log("calc first moment values for "+p)
8✔
481
            if include_first_moment:
8✔
482
                mean_df = pd.DataFrame(pt_mean,index=reals)
8✔
483
                mean_dfs[group] = mean_df
8✔
484
            var_data[group] = pt_var
8✔
485

486
            self.logger.log("processing " + group)
8✔
487

488
        dfstd = pd.DataFrame(var_data,index=self.predictions).apply(np.sqrt).T
8✔
489
        dfper = dfstd.copy()
8✔
490
        prior_std = prior_var.apply(np.sqrt)
8✔
491
        for p in self.predictions:
8✔
492
            dfper.loc[groups,p] = 100 * (1-(dfstd.loc[groups,p].values/prior_std.loc[p]))
8✔
493
        dfper = dfper.loc[groups,self.predictions]
8✔
494

495
        return mean_dfs,dfstd,dfper
8✔
496

497

498
    def prep_for_dsi(self,sim_ensemble=None,t_d="dsi_template"):
9✔
499
        """setup a new PEST interface for the data-space inversion process
500

501
        Args:
502

503
            sim_ensemble (`pyemu.ObservationEnsemble`): observation ensemble to use for DSI latent space
504
                variables.  If `None`, use `self.sim_ensemble`.  Default is `None`
505
            t_d (`str`): template directory to setup the DSI model + pest files in.  Default is `dsi_template`
506

507

508
        Example::
509

510
        #assumes "my.pst" exists
511
        ends = pyemu.EnDS(ensemble="my.0.obs.jcb",forecasts=["fore1","fore2"])
512
        ends.prep_for_dsi() #setup a new pest interface() based on the DSI approach
513
        pyemu.os_utils.start_workers("pestpp-ies","my.pst","dsi_template",num_workers=20,
514
                                      master_dir="dsi_master")
515
                                      
516

517

518
        """
519
        if sim_ensemble is None:
8✔
520
            sim_ensemble = self.sim_ensemble.copy()
8✔
521

522
        if os.path.exists(t_d):
8✔
523
            self.logger.warn("EnDS.prep_for_dsi(): t_d '{0}' exists, removing...".format(t_d))
×
524
            shutil.rmtree(t_d)
×
525
        os.makedirs(t_d)
8✔
526

527
        self.logger.log("getting deviations")
8✔
528
        nz_names = self.pst.nnz_obs_names
8✔
529
        snz_names = set(nz_names)
8✔
530
        z_names = [n for n in self.pst.obs_names if n not in snz_names]
8✔
531
        names = z_names.copy()
8✔
532
        names.extend(nz_names)
8✔
533
        names.sort()
8✔
534
        oe = sim_ensemble.get_deviations() / np.sqrt(float(sim_ensemble.shape[0] - 1))
8✔
535
        oe = oe.loc[:,names]
8✔
536

537
        self.logger.log("getting deviations")
8✔
538

539
        self.logger.log("pseudo inv of deviations matrix")
8✔
540

541
        #deltad = Matrix.from_dataframe(oe).T
542
        #U,S,V = deltad.pseudo_inv_components(maxsing=oe.shape[0],eigthresh=1e-30)
543

544
        U, S, V = np.linalg.svd(oe.values, full_matrices=False)
8✔
545
        self.logger.log("pseudo inv of deviations matrix")
8✔
546

547

548
        self.logger.log("creating tpl files")
8✔
549
        dsi_in_file = os.path.join(t_d, "dsi_pars.csv")
8✔
550
        dsi_tpl_file = dsi_in_file + ".tpl"
8✔
551
        ftpl = open(dsi_tpl_file, 'w')
8✔
552
        fin = open(dsi_in_file, 'w')
8✔
553
        ftpl.write("ptf ~\n")
8✔
554
        fin.write("parnme,parval1\n")
8✔
555
        ftpl.write("parnme,parval1\n")
8✔
556
        npar = S.shape[0]
8✔
557
        dsi_pnames = []
8✔
558
        for i in range(npar):
8✔
559
            pname = "dsi_par{0:04d}".format(i)
8✔
560
            dsi_pnames.append(pname)
8✔
561
            fin.write("{0},0.0\n".format(pname))
8✔
562
            ftpl.write("{0},~   {0}   ~\n".format(pname, pname))
8✔
563
        fin.close()
8✔
564
        ftpl.close()
8✔
565

566
        mn_vec = sim_ensemble.mean(axis=0)
8✔
567
        mn_in_file = os.path.join(t_d, "dsi_pr_mean.csv")
8✔
568
        mn_tpl_file = mn_in_file + ".tpl"
8✔
569
        fin = open(mn_in_file, 'w')
8✔
570
        ftpl = open(mn_tpl_file, 'w')
8✔
571
        ftpl.write("ptf ~\n")
8✔
572
        fin.write("obsnme,mn\n")
8✔
573
        ftpl.write("obsnme,pn\n")
8✔
574
        mn_dict = {}
8✔
575
        for oname in names:
8✔
576
            pname = "dsi_prmn_{0}".format(oname)
8✔
577
            fin.write("{0},{1}\n".format(oname, mn_vec[oname]))
8✔
578
            ftpl.write("{0},~   {1}   ~\n".format(oname, pname))
8✔
579
            mn_dict[pname] = mn_vec[oname]
8✔
580
        fin.close()
8✔
581
        ftpl.close()
8✔
582
        self.logger.log("creating tpl files")
8✔
583

584
        self.logger.log("saving proj mat")
8✔
585

586
        #pmat = U * S
587
        pmat = np.dot(V.transpose(), np.diag(S))
8✔
588
        #row_names = ["sing_vec_{0}".format(i) for i in range(pmat.shape[0])]
589
        pmat = Matrix(x=pmat,col_names=dsi_pnames,row_names=names)
8✔
590
        pmat.col_names = dsi_pnames
8✔
591
        proj_name = "dsi_proj_mat.jcb" # dont change this name!!!
8✔
592
        proj_path = os.path.join(t_d,proj_name)
8✔
593
        pmat.to_coo(proj_path)
8✔
594
        self.logger.statement("projection matrix dimensions:"+str(pmat.shape))
8✔
595
        self.logger.statement("projection matrix saved to "+proj_path)
8✔
596
        self.logger.log("saving proj mat")
8✔
597

598

599

600

601
        # this is the dsi forward run function - it is harded coded below!
602
        def dsi_forward_run():
8✔
603
            import numpy as np
8✔
604
            import pandas as pd
8✔
605
            import pyemu
8✔
606
            pmat = pyemu.Matrix.from_binary("dsi_proj_mat.jcb")
8✔
607
            pvals = pd.read_csv("dsi_pars.csv",index_col=0)
8✔
608
            pvals = pvals.loc[pmat.col_names,:]
8✔
609
            ovals = pd.read_csv("dsi_pr_mean.csv",index_col=0)
8✔
610
            ovals = ovals.loc[pmat.row_names,:]
8✔
611
            sim_vals = ovals + np.dot(pmat.x,pvals.values)
8✔
612
            print(sim_vals)
8✔
613
            sim_vals.to_csv("dsi_sim_vals.csv")
8✔
614

615
        self.logger.log("test run")
8✔
616
        b_d = os.getcwd()
8✔
617
        os.chdir(t_d)
8✔
618
        dsi_forward_run()
8✔
619
        os.chdir(b_d)
8✔
620
        self.logger.log("test run")
8✔
621

622
        self.logger.log("creating ins file")
8✔
623
        out_file = os.path.join(t_d,"dsi_sim_vals.csv")
8✔
624
        ins_file = out_file + ".ins"
8✔
625
        sdf = pd.read_csv(out_file,index_col=0)
8✔
626
        with open(ins_file,'w') as f:
8✔
627
            f.write("pif ~\n")
8✔
628
            f.write("l1\n")
8✔
629
            for oname in sdf.index.values:
8✔
630
                f.write("l1 ~,~ !{0}!\n".format(oname))
8✔
631
        self.logger.log("creating ins file")
8✔
632

633
        self.logger.log("creating Pst")
8✔
634
        pst = Pst.from_io_files([mn_tpl_file,dsi_tpl_file],[mn_in_file,dsi_in_file],[ins_file],[out_file],pst_path=".")
8✔
635

636
        par = pst.parameter_data
8✔
637
        dsi_pars = par.loc[par.parnme.str.startswith("dsi_par"),"parnme"]
8✔
638
        par.loc[dsi_pars,"parval1"] = 0
8✔
639
        par.loc[dsi_pars,"parubnd"] = 2.5
8✔
640
        par.loc[dsi_pars,"parlbnd"] = -2.5
8✔
641
        par.loc[dsi_pars,"partrans"] = "none"
8✔
642
        mn_pars = par.loc[par.parnme.str.startswith("dsi_prmn"),"parnme"]
8✔
643
        par.loc[mn_pars,"partrans"] = "fixed"
8✔
644
        for pname,pval in mn_dict.items():
8✔
645
            par.loc[pname,"parval1"] = pval
8✔
646
            par.loc[pname, "parubnd"] = pval + 1000
8✔
647
            par.loc[pname, "parlbnd"] = pval - 1000
8✔
648

649
        obs = pst.observation_data
8✔
650
        org_obs = self.pst.observation_data
8✔
651
        for col in org_obs.columns:
8✔
652
            obs.loc[org_obs.obsnme,col] = org_obs.loc[:,col]
8✔
653
        pst.control_data.noptmax = 0
8✔
654
        pst.model_command = "python forward_run.py"
8✔
655
        self.logger.log("creating Pst")
8✔
656
        import inspect
8✔
657
        lines = [line[12:] for line in inspect.getsource(dsi_forward_run).split("\n")][1:]
8✔
658

659
        with open(os.path.join(t_d,"forward_run.py"),'w') as f:
8✔
660
            
661
            for line in lines:
8✔
662
                f.write(line+"\n")
8✔
663
        pst.write(os.path.join(t_d,"dsi.pst"),version=2)
8✔
664
        self.logger.statement("saved pst to {0}".format(os.path.join(t_d,"dsi.pst")))
8✔
665
        try:
8✔
666
            run("pestpp-ies dsi.pst",cwd=t_d)
8✔
667
        except Exception as e:
×
668
            self.logger.warn("error testing noptmax=0 run:{0}".format(str(e)))
×
669

670
        return pst
8✔
671

672

673

674

675

676

677
        
678

679

680

681

682

683

STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc