20220908.32

Build # 20220908.32

Build Type

push

circleci

Committed by GitHub

Commit Message

<a href="https://github.com/great-expectations/great_expectations/commit/<a class=hub.com/great-expectations/great_expectations/commit/ed12b54772bf54cba6a371119ae34123d815e4c8">ed12b5477<a href="https://github.com/great-expectations/great_expectations/commit/ed12b54772bf54cba6a371119ae34123d815e4c8">">Merge </a><a class="double-link" href="https://github.com/great-expectations/great_expectations/commit/<a class="double-link" href="https://github.com/great-expectations/great_expectations/commit/1cbbd41a5ef54457794f75e3b7b5a43ace8d4c28">1cbbd41a5</a>">1cbbd41a5</a><a href="https://github.com/great-expectations/great_expectations/commit/ed12b54772bf54cba6a371119ae34123d815e4c8"> into 0cea1b9b1">0cea1b9b1</a>

Run Details

18855 of 49081 relevant lines covered (38.42%)

1.15 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

50.0

/great_expectations/rule_based_profiler/parameter_builder/regex_pattern_string_parameter_builder.py

import logging
from typing import Dict, Iterable, List, Optional, Set, Union

import great_expectations.exceptions as ge_exceptions
from great_expectations.rule_based_profiler.attributed_resolved_metrics import (
    AttributedResolvedMetrics,
)
from great_expectations.rule_based_profiler.config import ParameterBuilderConfig
from great_expectations.rule_based_profiler.domain import Domain
from great_expectations.rule_based_profiler.helpers.util import (
    NP_EPSILON,
    get_parameter_value_and_validate_return_type,
)
from great_expectations.rule_based_profiler.metric_computation_result import (
    MetricComputationResult,
    MetricValues,
)
from great_expectations.rule_based_profiler.parameter_builder import ParameterBuilder
from great_expectations.rule_based_profiler.parameter_container import (
    FULLY_QUALIFIED_PARAMETER_NAME_METADATA_KEY,
    FULLY_QUALIFIED_PARAMETER_NAME_VALUE_KEY,
    ParameterContainer,
)
from great_expectations.types.attributes import Attributes

logger = logging.getLogger(__name__)


class RegexPatternStringParameterBuilder(ParameterBuilder):
    """
    Detects the domain REGEX from a set of candidate REGEX strings by computing the
    column_values.match_regex_format.unexpected_count metric for each candidate format and returning the format that
    has the lowest unexpected_count ratio.
    """

    # list of candidate strings that are most commonly used
    # source: https://regexland.com/most-common-regular-expressions/
    # source for UUID: https://stackoverflow.com/questions/7905929/how-to-test-valid-uuid-guid/13653180#13653180
    CANDIDATE_REGEX: Set[str] = {
        r"\d+",  # whole number with 1 or more digits
        r"-?\d+",  # negative whole numbers
        r"-?\d+(?:\.\d*)?",  # decimal numbers with . (period) separator
        r"[A-Za-z0-9\.,;:!?()\"'%\-]+",  # general text
        r"^\s+",  # leading space
        r"\s+$",  # trailing space
        r"https?:\/\/(?:www\.)?[-a-zA-Z0-9@:%._\+~#=]{2,255}\.[a-z]{2,6}\b(?:[-a-zA-Z0-9@:%_\+.~#()?&//=]*)",  # Matching URL (including http(s) protocol)
        r"<\/?(?:p|a|b|img)(?: \/)?>",  # HTML tags
        r"(?:25[0-5]|2[0-4]\d|[01]\d{2}|\d{1,2})(?:.(?:25[0-5]|2[0-4]\d|[01]\d{2}|\d{1,2})){3}",  # IPv4 IP address
        r"(?:[A-Fa-f0-9]){0,4}(?: ?:? ?(?:[A-Fa-f0-9]){0,4}){0,7}",  # IPv6 IP address,
        r"\b[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}-[0-5][0-9a-fA-F]{3}-[089ab][0-9a-fA-F]{3}-\b[0-9a-fA-F]{12}\b ",  # UUID
    }

    def __init__(
        self,
        name: str,
        metric_domain_kwargs: Optional[Union[str, dict]] = None,
        metric_value_kwargs: Optional[Union[str, dict]] = None,
        threshold: Union[str, float] = 1.0,
        candidate_regexes: Optional[Union[str, Iterable[str]]] = None,
        evaluation_parameter_builder_configs: Optional[
            List[ParameterBuilderConfig]
        ] = None,
        data_context: Optional["BaseDataContext"] = None,  # noqa: F821
    ) -> None:
        """
        Configure this RegexPatternStringParameterBuilder
        Args:
            name: the name of this parameter -- this is user-specified parameter name (from configuration);
            it is not the fully-qualified parameter name; a fully-qualified parameter name must start with "$parameter."
            and may contain one or more subsequent parts (e.g., "$parameter.<my_param_from_config>.<metric_name>").
            threshold: the ratio of values that must match a format string for it to be accepted
            candidate_regexes: a list of candidate regex strings that will REPLACE the default
            evaluation_parameter_builder_configs: ParameterBuilder configurations, executing and making whose respective
            ParameterBuilder objects' outputs available (as fully-qualified parameter names) is pre-requisite.
            These "ParameterBuilder" configurations help build parameters needed for this "ParameterBuilder".
            data_context: BaseDataContext associated with this ParameterBuilder
        """
        super().__init__(
            name=name,
            evaluation_parameter_builder_configs=evaluation_parameter_builder_configs,
            data_context=data_context,
        )

        self._metric_domain_kwargs = metric_domain_kwargs
        self._metric_value_kwargs = metric_value_kwargs

        self._threshold = threshold

        self._candidate_regexes = candidate_regexes

    @property
    def metric_domain_kwargs(self) -> Optional[Union[str, dict]]:
        return self._metric_domain_kwargs

    @property
    def metric_value_kwargs(self) -> Optional[Union[str, dict]]:
        return self._metric_value_kwargs

    @metric_value_kwargs.setter
    def metric_value_kwargs(self, value: Optional[Union[str, dict]]) -> None:
        self._metric_value_kwargs = value

    @property
    def threshold(self) -> Union[str, float]:
        return self._threshold

    @property
    def candidate_regexes(
        self,
    ) -> Union[str, Union[List[str], Set[str]]]:
        return self._candidate_regexes

    def _build_parameters(
        self,
        domain: Domain,
        variables: Optional[ParameterContainer] = None,
        parameters: Optional[Dict[str, ParameterContainer]] = None,
        recompute_existing_parameter_values: bool = False,
    ) -> Attributes:
        """
        Builds ParameterContainer object that holds ParameterNode objects with attribute name-value pairs and details.

        Check the percentage of values matching the REGEX string, and return the best fit, or None if no string exceeds
        the configured threshold.

        Builds ParameterContainer object that holds ParameterNode objects with attribute name-value pairs and details.
        """
        metric_computation_result: MetricComputationResult

        metric_computation_result = self.get_metrics(
            metric_name="column_values.nonnull.count",
            metric_domain_kwargs=self.metric_domain_kwargs,
            metric_value_kwargs=self.metric_value_kwargs,
            domain=domain,
            variables=variables,
            parameters=parameters,
        )

        # This should never happen.
        if len(metric_computation_result.attributed_resolved_metrics) != 1:
            raise ge_exceptions.ProfilerExecutionError(
                message=f'Result of metric computations for {self.__class__.__name__} must be a list with exactly 1 element of type "AttributedResolvedMetrics" ({metric_computation_result.attributed_resolved_metrics} found).'
            )

        attributed_resolved_metrics: AttributedResolvedMetrics

        attributed_resolved_metrics = (
            metric_computation_result.attributed_resolved_metrics[0]
        )

        metric_values: MetricValues

        metric_values = attributed_resolved_metrics.conditioned_metric_values

        if metric_values is None:
            raise ge_exceptions.ProfilerExecutionError(
                message=f"Result of metric computations for {self.__class__.__name__} is empty."
            )

        # Now obtain 1-dimensional vector of values of computed metric (each element corresponds to a Batch ID).
        metric_values = metric_values[:, 0]

        nonnull_count: int = sum(metric_values)

        # Obtain candidate_regexes from "rule state" (i.e, variables and parameters); from instance variable otherwise.
        candidate_regexes: Union[
            List[str],
            Set[str],
        ] = get_parameter_value_and_validate_return_type(
            domain=domain,
            parameter_reference=self.candidate_regexes,
            expected_return_type=None,
            variables=variables,
            parameters=parameters,
        )
        if candidate_regexes is not None and isinstance(candidate_regexes, list):
            candidate_regexes = set(candidate_regexes)
        else:
            candidate_regexes = RegexPatternStringParameterBuilder.CANDIDATE_REGEX

        # Gather "metric_value_kwargs" for all candidate "regex" strings.
        regex_string: str
        match_regex_metric_value_kwargs_list: List[dict] = []
        match_regex_metric_value_kwargs: dict
        for regex_string in candidate_regexes:
            if self.metric_value_kwargs:
                match_regex_metric_value_kwargs: dict = {
                    **self._metric_value_kwargs,
                    **{"regex": regex_string},
                }
            else:
                match_regex_metric_value_kwargs = {
                    "regex": regex_string,
                }

            match_regex_metric_value_kwargs_list.append(match_regex_metric_value_kwargs)

        # Obtain resolved metrics and metadata for all metric configurations and available Batch objects simultaneously.
        metric_computation_result = self.get_metrics(
            metric_name="column_values.match_regex.unexpected_count",
            metric_domain_kwargs=self.metric_domain_kwargs,
            metric_value_kwargs=match_regex_metric_value_kwargs_list,
            domain=domain,
            variables=variables,
            parameters=parameters,
        )

        regex_string_success_ratios: dict = {}

        for (
            attributed_resolved_metrics
        ) in metric_computation_result.attributed_resolved_metrics:
            # Now obtain 1-dimensional vector of values of computed metric (each element corresponds to a Batch ID).
            metric_values = attributed_resolved_metrics.conditioned_metric_values[:, 0]

            match_regex_unexpected_count: int = sum(metric_values)
            success_ratio: float = (nonnull_count - match_regex_unexpected_count) / (
                nonnull_count + NP_EPSILON
            )
            regex_string_success_ratios[
                attributed_resolved_metrics.metric_attributes["regex"]
            ] = success_ratio

        # Obtain threshold from "rule state" (i.e., variables and parameters); from instance variable otherwise.
        threshold: float = get_parameter_value_and_validate_return_type(
            domain=domain,
            parameter_reference=self._threshold,
            expected_return_type=float,
            variables=variables,
            parameters=parameters,
        )

        # get best-matching regex_string that match greater than threshold
        (
            best_regex_string,
            best_ratio,
        ) = ParameterBuilder._get_best_candidate_above_threshold(
            regex_string_success_ratios, threshold
        )
        # dict of sorted regex and ratios for all evaluated candidates
        sorted_regex_candidates_and_ratios: dict = (
            ParameterBuilder._get_sorted_candidates_and_ratios(
                regex_string_success_ratios
            )
        )

        return Attributes(
            {
                FULLY_QUALIFIED_PARAMETER_NAME_VALUE_KEY: best_regex_string,
                FULLY_QUALIFIED_PARAMETER_NAME_METADATA_KEY: {
                    "success_ratio": best_ratio,
                    "evaluated_regexes": sorted_regex_candidates_and_ratios,
                },
            }
        )

1	import logging	3✔
2	from typing import Dict, Iterable, List, Optional, Set, Union	3✔
3
4	import great_expectations.exceptions as ge_exceptions	3✔
5	from great_expectations.rule_based_profiler.attributed_resolved_metrics import (	3✔
6	AttributedResolvedMetrics,
7	)
8	from great_expectations.rule_based_profiler.config import ParameterBuilderConfig	3✔
9	from great_expectations.rule_based_profiler.domain import Domain	3✔
10	from great_expectations.rule_based_profiler.helpers.util import (	3✔
11	NP_EPSILON,
12	get_parameter_value_and_validate_return_type,
13	)
14	from great_expectations.rule_based_profiler.metric_computation_result import (	3✔
15	MetricComputationResult,
16	MetricValues,
17	)
18	from great_expectations.rule_based_profiler.parameter_builder import ParameterBuilder	3✔
19	from great_expectations.rule_based_profiler.parameter_container import (	3✔
20	FULLY_QUALIFIED_PARAMETER_NAME_METADATA_KEY,
21	FULLY_QUALIFIED_PARAMETER_NAME_VALUE_KEY,
22	ParameterContainer,
23	)
24	from great_expectations.types.attributes import Attributes	3✔
25
26	logger = logging.getLogger(__name__)	3✔
27
28
29	class RegexPatternStringParameterBuilder(ParameterBuilder):	3✔
30	"""
31	Detects the domain REGEX from a set of candidate REGEX strings by computing the
32	column_values.match_regex_format.unexpected_count metric for each candidate format and returning the format that
33	has the lowest unexpected_count ratio.
34	"""
35
36	# list of candidate strings that are most commonly used
37	# source: https://regexland.com/most-common-regular-expressions/
38	# source for UUID: https://stackoverflow.com/questions/7905929/how-to-test-valid-uuid-guid/13653180#13653180
39	CANDIDATE_REGEX: Set[str] = {	3✔
40	r"\d+", # whole number with 1 or more digits
41	r"-?\d+", # negative whole numbers
42	r"-?\d+(?:\.\d*)?", # decimal numbers with . (period) separator
43	r"[A-Za-z0-9\.,;:!?()\"'%\-]+", # general text
44	r"^\s+", # leading space
45	r"\s+$", # trailing space
46	r"https?:\/\/(?:www\.)?[-a-zA-Z0-9@:%._\+~#=]{2,255}\.[a-z]{2,6}\b(?:[-a-zA-Z0-9@:%_\+.~#()?&//=]*)", # Matching URL (including http(s) protocol)
47	r"<\/?(?:p\|a\|b\|img)(?: \/)?>", # HTML tags
48	r"(?:25[0-5]\|2[0-4]\d\|[01]\d{2}\|\d{1,2})(?:.(?:25[0-5]\|2[0-4]\d\|[01]\d{2}\|\d{1,2})){3}", # IPv4 IP address
49	r"(?:[A-Fa-f0-9]){0,4}(?: ?:? ?(?:[A-Fa-f0-9]){0,4}){0,7}", # IPv6 IP address,
50	r"\b[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}-[0-5][0-9a-fA-F]{3}-[089ab][0-9a-fA-F]{3}-\b[0-9a-fA-F]{12}\b ", # UUID
51	}
52
53	def __init__(	3✔
54	self,
55	name: str,
56	metric_domain_kwargs: Optional[Union[str, dict]] = None,
57	metric_value_kwargs: Optional[Union[str, dict]] = None,
58	threshold: Union[str, float] = 1.0,
59	candidate_regexes: Optional[Union[str, Iterable[str]]] = None,
60	evaluation_parameter_builder_configs: Optional[
61	List[ParameterBuilderConfig]
62	] = None,
63	data_context: Optional["BaseDataContext"] = None, # noqa: F821
64	) -> None:
65	"""
66	Configure this RegexPatternStringParameterBuilder
67	Args:
68	name: the name of this parameter -- this is user-specified parameter name (from configuration);
69	it is not the fully-qualified parameter name; a fully-qualified parameter name must start with "$parameter."
70	and may contain one or more subsequent parts (e.g., "$parameter.<my_param_from_config>.<metric_name>").
71	threshold: the ratio of values that must match a format string for it to be accepted
72	candidate_regexes: a list of candidate regex strings that will REPLACE the default
73	evaluation_parameter_builder_configs: ParameterBuilder configurations, executing and making whose respective
74	ParameterBuilder objects' outputs available (as fully-qualified parameter names) is pre-requisite.
75	These "ParameterBuilder" configurations help build parameters needed for this "ParameterBuilder".
76	data_context: BaseDataContext associated with this ParameterBuilder
77	"""
78	super().__init__(	3✔
79	name=name,
80	evaluation_parameter_builder_configs=evaluation_parameter_builder_configs,
81	data_context=data_context,
82	)
83
84	self._metric_domain_kwargs = metric_domain_kwargs	3✔
85	self._metric_value_kwargs = metric_value_kwargs	3✔
86
87	self._threshold = threshold	3✔
88
89	self._candidate_regexes = candidate_regexes	3✔
90
91	@property	3✔
92	def metric_domain_kwargs(self) -> Optional[Union[str, dict]]:	3✔
93	return self._metric_domain_kwargs	×
94
95	@property	3✔
96	def metric_value_kwargs(self) -> Optional[Union[str, dict]]:	3✔
97	return self._metric_value_kwargs	×
98
99	@metric_value_kwargs.setter	3✔
100	def metric_value_kwargs(self, value: Optional[Union[str, dict]]) -> None:	3✔
101	self._metric_value_kwargs = value	×
102
103	@property	3✔
104	def threshold(self) -> Union[str, float]:	3✔
105	return self._threshold	3✔
106
107	@property	3✔
108	def candidate_regexes(	3✔
109	self,
110	) -> Union[str, Union[List[str], Set[str]]]:
111	return self._candidate_regexes	3✔
112
113	def _build_parameters(	3✔
114	self,
115	domain: Domain,
116	variables: Optional[ParameterContainer] = None,
117	parameters: Optional[Dict[str, ParameterContainer]] = None,
118	recompute_existing_parameter_values: bool = False,
119	) -> Attributes:
120	"""
121	Builds ParameterContainer object that holds ParameterNode objects with attribute name-value pairs and details.
122
123	Check the percentage of values matching the REGEX string, and return the best fit, or None if no string exceeds
124	the configured threshold.
125
126	Builds ParameterContainer object that holds ParameterNode objects with attribute name-value pairs and details.
127	"""
128	metric_computation_result: MetricComputationResult
129
130	metric_computation_result = self.get_metrics(	×
131	metric_name="column_values.nonnull.count",
132	metric_domain_kwargs=self.metric_domain_kwargs,
133	metric_value_kwargs=self.metric_value_kwargs,
134	domain=domain,
135	variables=variables,
136	parameters=parameters,
137	)
138
139	# This should never happen.
140	if len(metric_computation_result.attributed_resolved_metrics) != 1:	×
141	raise ge_exceptions.ProfilerExecutionError(	×
142	message=f'Result of metric computations for {self.__class__.__name__} must be a list with exactly 1 element of type "AttributedResolvedMetrics" ({metric_computation_result.attributed_resolved_metrics} found).'
143	)
144
145	attributed_resolved_metrics: AttributedResolvedMetrics
146
147	attributed_resolved_metrics = (	×
148	metric_computation_result.attributed_resolved_metrics[0]
149	)
150
151	metric_values: MetricValues
152
153	metric_values = attributed_resolved_metrics.conditioned_metric_values	×
154
155	if metric_values is None:	×
156	raise ge_exceptions.ProfilerExecutionError(	×
157	message=f"Result of metric computations for {self.__class__.__name__} is empty."
158	)
159
160	# Now obtain 1-dimensional vector of values of computed metric (each element corresponds to a Batch ID).
161	metric_values = metric_values[:, 0]	×
162
163	nonnull_count: int = sum(metric_values)	×
164
165	# Obtain candidate_regexes from "rule state" (i.e, variables and parameters); from instance variable otherwise.
166	candidate_regexes: Union[	×
167	List[str],
168	Set[str],
169	] = get_parameter_value_and_validate_return_type(
170	domain=domain,
171	parameter_reference=self.candidate_regexes,
172	expected_return_type=None,
173	variables=variables,
174	parameters=parameters,
175	)
176	if candidate_regexes is not None and isinstance(candidate_regexes, list):	×
177	candidate_regexes = set(candidate_regexes)	×
178	else:
179	candidate_regexes = RegexPatternStringParameterBuilder.CANDIDATE_REGEX	×
180
181	# Gather "metric_value_kwargs" for all candidate "regex" strings.
182	regex_string: str
183	match_regex_metric_value_kwargs_list: List[dict] = []	×
184	match_regex_metric_value_kwargs: dict
185	for regex_string in candidate_regexes:	×
186	if self.metric_value_kwargs:	×
187	match_regex_metric_value_kwargs: dict = {	×
188	**self._metric_value_kwargs,
189	**{"regex": regex_string},
190	}
191	else:
192	match_regex_metric_value_kwargs = {	×
193	"regex": regex_string,
194	}
195
196	match_regex_metric_value_kwargs_list.append(match_regex_metric_value_kwargs)	×
197
198	# Obtain resolved metrics and metadata for all metric configurations and available Batch objects simultaneously.
199	metric_computation_result = self.get_metrics(	×
200	metric_name="column_values.match_regex.unexpected_count",
201	metric_domain_kwargs=self.metric_domain_kwargs,
202	metric_value_kwargs=match_regex_metric_value_kwargs_list,
203	domain=domain,
204	variables=variables,
205	parameters=parameters,
206	)
207
208	regex_string_success_ratios: dict = {}	×
209
210	for (	×
211	attributed_resolved_metrics
212	) in metric_computation_result.attributed_resolved_metrics:
213	# Now obtain 1-dimensional vector of values of computed metric (each element corresponds to a Batch ID).
214	metric_values = attributed_resolved_metrics.conditioned_metric_values[:, 0]	×
215
216	match_regex_unexpected_count: int = sum(metric_values)	×
217	success_ratio: float = (nonnull_count - match_regex_unexpected_count) / (	×
218	nonnull_count + NP_EPSILON
219	)
220	regex_string_success_ratios[	×
221	attributed_resolved_metrics.metric_attributes["regex"]
222	] = success_ratio
223
224	# Obtain threshold from "rule state" (i.e., variables and parameters); from instance variable otherwise.
225	threshold: float = get_parameter_value_and_validate_return_type(	×
226	domain=domain,
227	parameter_reference=self._threshold,
228	expected_return_type=float,
229	variables=variables,
230	parameters=parameters,
231	)
232
233	# get best-matching regex_string that match greater than threshold
234	(	×
235	best_regex_string,
236	best_ratio,
237	) = ParameterBuilder._get_best_candidate_above_threshold(
238	regex_string_success_ratios, threshold
239	)
240	# dict of sorted regex and ratios for all evaluated candidates
241	sorted_regex_candidates_and_ratios: dict = (	×
242	ParameterBuilder._get_sorted_candidates_and_ratios(
243	regex_string_success_ratios
244	)
245	)
246
247	return Attributes(	×
248	{
249	FULLY_QUALIFIED_PARAMETER_NAME_VALUE_KEY: best_regex_string,
250	FULLY_QUALIFIED_PARAMETER_NAME_METADATA_KEY: {
251	"success_ratio": best_ratio,
252	"evaluated_regexes": sorted_regex_candidates_and_ratios,
253	},
254	}
255	)

great-expectations / great_expectations / 20220908.32

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous