16573271716

Committed 28 Jul 2025 03:29PM UTC coverage: 69.438% (+0.6%) from 68.803%

Build # 16573271716

Build Type

Pull #9

github

Committed by

web-flow

Commit Message

Merge 227c6fb62 into 30e6d4242

Pull Request Pull Request #9: Improve string template

Run Details

158 of 170 new or added lines in 9 files covered. (92.94%)

27 existing lines in 2 files now uncovered.

4969 of 7156 relevant lines covered (69.44%)

0.92 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.18

/internal/generator/usecase/general/generator/value/string.go

package value

import (
        "math"
        "math/big"
        "regexp"
        "slices"
        "strings"

        "github.com/flosch/pongo2"
        "github.com/pkg/errors"
        "github.com/tarantool/sdvg/internal/generator/common"
        "github.com/tarantool/sdvg/internal/generator/models"
        "github.com/tarantool/sdvg/internal/generator/usecase/general/locale"
        "github.com/tarantool/sdvg/internal/generator/usecase/general/locale/en"
        "github.com/tarantool/sdvg/internal/generator/usecase/general/locale/ru"
)

var (
        rePatternVal = regexp.MustCompile(`pattern\((?:'([^']*)'|"([^"]*)")\)`)
)

// Verify interface compliance in compile time.
var _ Generator = (*StringGenerator)(nil)

// StringGenerator type is used to describe generator for strings.
type StringGenerator struct {
        *models.ColumnStringParams
        totalValuesCount uint64
        localeModule     locale.LocalModule
        template         *pongo2.Template
        charset          []rune
        countByPrefix    []float64
        sumByPrefix      []float64
        completions      []int64 // completions[i] stores the number of ways to form a text of length i
}

//nolint:cyclop
func (g *StringGenerator) Prepare() error {
        if g.Template != "" {
                template, err := pongo2.FromString(g.Template)
                if err != nil {
                        return errors.Errorf("failed to parse template: %s", err.Error())
                }

                g.template = template
        }

        switch g.Locale {
        case "ru":
                g.localeModule = ru.NewLocaleModule(g.LogicalType, g.MinLength, g.MaxLength)
        case "en":
                g.localeModule = en.NewLocaleModule(g.LogicalType, g.MinLength, g.MaxLength)
        default:
                return errors.Errorf("unknown locale: %q", g.Locale)
        }

        switch g.LogicalType {
        case models.FirstNameType:
                if len(g.localeModule.GetFirstNames(locale.MaleGender)) == 0 {
                        return errors.Errorf("no male first names with length between %v and %v", g.MinLength, g.MaxLength)
                }

                if len(g.localeModule.GetFirstNames(locale.FemaleGender)) == 0 {
                        return errors.Errorf("no female first names with length between %v and %v", g.MinLength, g.MaxLength)
                }
        case models.LastNameType:
                if len(g.localeModule.GetLastNames(locale.MaleGender)) == 0 {
                        return errors.Errorf("no male last names with length between %v and %v", g.MinLength, g.MaxLength)
                }

                if len(g.localeModule.GetLastNames(locale.FemaleGender)) == 0 {
                        return errors.Errorf("no female last names with length between %v and %v", g.MinLength, g.MaxLength)
                }
        case models.PhoneType:
                if len(g.localeModule.GetPhonePatterns()) == 0 {
                        return errors.Errorf("no phone patterns with length between %v and %v", g.MinLength, g.MaxLength)
                }
        }

        g.charset = make([]rune, 0)

        if !g.WithoutLargeLetters {
                g.charset = append(g.charset, g.localeModule.LargeLetters()...)
        }

        if !g.WithoutSmallLetters {
                g.charset = append(g.charset, g.localeModule.SmallLetters()...)
        }

        if !g.WithoutNumbers {
                g.charset = append(g.charset, locale.Numbers...)
        }

        if !g.WithoutSpecialChars {
                g.charset = append(g.charset, locale.SpecialChars...)
        }

        slices.Sort(g.charset)

        if g.LogicalType == models.TextType {
                g.completions = g.calculateCompletions(g.MaxLength + 1)
        }

        return nil
}

func (g *StringGenerator) SetTotalCount(totalValuesCount uint64) error {
        g.totalValuesCount = totalValuesCount

        if g.LogicalType == "" && g.Template == "" {
                countByLength := make([]float64, g.MaxLength+1)
                avgRangeCount := math.Ceil(float64(totalValuesCount) / float64(g.MaxLength-g.MinLength+1))

                for length := g.MinLength; length <= g.MaxLength; length++ {
                        rangeCount := math.Pow(float64(len(g.charset)), float64(length))

                        var currentLenCount float64
                        if avgRangeCount > rangeCount {
                                currentLenCount = rangeCount
                                avgRangeCount += (avgRangeCount - rangeCount) / float64(g.MaxLength-length)
                        } else {
                                currentLenCount = math.Ceil(avgRangeCount)
                        }

                        countByLength[length] = currentLenCount
                }

                g.countByPrefix = make([]float64, g.MaxLength+1)
                g.sumByPrefix = make([]float64, g.MaxLength+1)

                for prefix := 0; prefix <= g.MaxLength; prefix++ {
                        prefixDivider := math.Pow(float64(len(g.charset)), float64(prefix))
                        g.countByPrefix[prefix] = countByLength[prefix] / prefixDivider

                        for length := 0; length <= g.MaxLength-prefix; length++ {
                                g.sumByPrefix[prefix] += countByLength[length+prefix] / prefixDivider
                        }
                }
        }

        return nil
}

// calculateCompletions precomputes completions.
func (g *StringGenerator) calculateCompletions(length int) []int64 {
        words := g.localeModule.GetWords()
        bytesPerChar := g.localeModule.GetBytesPerChar()
        delimiterLen := len(locale.WordsDelimiter)

        completionsBig := make([]*big.Int, length+1)
        for i := range completionsBig {
                completionsBig[i] = big.NewInt(0)
        }

        // Base case: one way to form a text of length 0 (the empty text).
        completionsBig[0].SetInt64(1)

        // Base case: all one-letter words.
        for _, w := range words {
                if len(w) == 1 {
                        completionsBig[1].Add(completionsBig[1], big.NewInt(1))
                }
        }

        // For every target length, add ways by choosing each word that fits.
        for l := 2; l <= length; l++ {
                for _, w := range words {
                        wLen := len(w)/bytesPerChar + delimiterLen
                        if wLen <= l {
                                completionsBig[l].Add(completionsBig[l], completionsBig[l-wLen])
                        }
                }
        }

        // convert from big.Int to int64
        completions := make([]int64, 0, length+1)

        for _, blockCount := range completionsBig {
                if !blockCount.IsInt64() {
                        break
                }

                completions = append(completions, blockCount.Int64())
        }

        return completions
}

// templateString returns n-th string by template.
func (g *StringGenerator) templateString(number float64, rowValues map[string]any) (string, error) {
        if rowValues == nil {
                rowValues = make(map[string]any)
        }

        rowValues["pattern"] = func(pattern string) *pongo2.Value {
                return pongo2.AsSafeValue(g.patternString(number, pattern))
        }

        val, err := g.template.Execute(rowValues)
        if err != nil {
                return "", errors.New(err.Error())
        }

        return val, nil
}

// patternString returns n-th string by pattern.
func (g *StringGenerator) patternString(number float64, pattern string) string {
        val := []rune(pattern)
        index := number / float64(g.totalValuesCount)

        for i := range val {
                var letters []rune

                switch val[i] {
                case 'A':
                        letters = g.localeModule.LargeLetters()
                case 'a':
                        letters = g.localeModule.SmallLetters()
                case '0':
                        letters = locale.Numbers
                case '#':
                        letters = locale.SpecialChars
                default:
                        continue
                }

                var pos int
                pos, index = orderedPos(len(letters), index)
                val[i] = letters[pos]
        }

        return string(val)
}

// firstName returns n-th first name from range.
func (g *StringGenerator) firstName(number float64) string {
        firstNames := g.localeModule.GetFirstNames(locale.AnyGender)

        pos := orderedInt64(0, int64(len(firstNames)-1), number, g.totalValuesCount)

        return firstNames[pos]
}

// lastName returns n-th last name from range.
func (g *StringGenerator) lastName(number float64) string {
        lastNames := g.localeModule.GetLastNames(locale.AnyGender)

        pos := orderedInt64(0, int64(len(lastNames)-1), number, g.totalValuesCount)

        return lastNames[pos]
}

// phone returns n-th phone number from range.
func (g *StringGenerator) phone(number float64) string {
        patterns := g.localeModule.GetPhonePatterns()

        pos := orderedInt64(0, int64(len(patterns)-1), number, g.totalValuesCount)

        pattern := patterns[pos]
        maxPhone := int64(math.Pow(10, float64(strings.Count(pattern, "#")))) - 1 //nolint:mnd

        phone := orderedInt64(0, maxPhone, number, g.totalValuesCount)

        return replaceWithNumber(pattern, '#', phone)
}

// text sorts texts only within their respective length groups.
// Texts of the same length will be ordered, but ordering
// between texts of different lengths is not guaranteed.
//
//nolint:cyclop
func (g *StringGenerator) text(num float64) (string, error) {
        words := g.localeModule.GetWords()
        oneLetterWords := g.localeModule.GetOneLetterWords()
        oneLetterWordsLen := int64(len(oneLetterWords))

        delimiter := locale.WordsDelimiter
        delimiterLen := len(delimiter)

        bytesPerChar := g.localeModule.GetBytesPerChar()

        maxPreComputedLength := len(g.completions) - 1

        wantedLen := g.MinLength + delimiterLen + int(num)%(g.MaxLength-g.MinLength+1)

        number := int64(math.Floor(float64(g.completions[maxPreComputedLength]-1) * (num / float64(g.totalValuesCount))))

        result := make([]byte, 0, wantedLen*bytesPerChar)

        var textLen int

        remaining := maxPreComputedLength
        // Process until we've built the full text.
        for remaining > 0 {
                found := false
                // Iterate over words in lexicographical order.
                if remaining == 1 {
                        if number > oneLetterWordsLen-1 {
                                return "", errors.Errorf("remaining length is 1 but k: %v overflows: %v", number, oneLetterWordsLen)
                        }

                        result = append(result, oneLetterWords[number]...)

                        textLen++

                        break
                }

                for _, w := range words {
                        wLen := len(w)/bytesPerChar + delimiterLen
                        if wLen > remaining {
                                continue
                        }
                        // count = number of completions if we choose word w at this step.
                        count := g.completions[remaining-wLen]
                        // If k is within the block for word w, choose it.
                        if number < count {
                                result = append(result, w...)
                                result = append(result, delimiter...)

                                textLen += wLen

                                remaining -= wLen
                                found = true

                                break
                        }
                        // Otherwise, skip this block.
                        number -= count
                }

                if !found {
                        return "", errors.Errorf("index %v out of range for remaining length %d, %v", number, remaining, wantedLen)
                }
        }

        for textLen < wantedLen {
                w := words[number%int64(len(words)-1)]

                result = append(result, w...)
                result = append(result, delimiter...)

                textLen += len(w)/bytesPerChar + delimiterLen
        }

        text := string(result)

        if textLen > wantedLen {
                if bytesPerChar == 1 {
                        text = text[:wantedLen]
                } else {
                        text = string([]rune(text)[:wantedLen])
                }
        }

        return text, nil
}

// simpleString generates a lexicographically ordered string based on the given number.
// The function ensures that strings of different lengths are evenly distributed.
//
// Prepared variables (from Prepare method):
//   - countByLength - determines how many strings of each length should be generated; aims for an even distribution
//     but adjusts when the number of possible strings at a given length is limited;
//   - countByPrefix - determines how many times a given prefix should be repeated across generated strings;
//   - sumByPrefix - keeps total number of strings that should be generated with a specific prefix of a certain length.
//
// Each iteration of loop follows these steps:
//   - Subtracting the Current Prefix Group.
//     countByPrefix[prefixLen] represents how many times the current prefix is repeated.
//     We subtract this value from remain to determine if the target string falls within this group.
//     If remain is negative, it means the desired index falls within the current prefix group, so we stop.
//     If sumByPrefix[prefixLen+1] == 0, it means no further characters can be added, so we also stop.
//   - Determining the Next Character.
//     sumByPrefix[prefixLen+1] tells us how many strings exist for the next character choices.
//     remain / sumByPrefix[prefixLen+1] determines how many prefixes we need to skip before choosing next character.
//     We update remain according to reflect the choice. The selected character charset[i] is added to prefix.
//
// This approach ensures precision up to 217 characters in prefix length due to float64 limitations.
// Any additional characters required beyond the ordered prefix are filled in using a pattern based on `number`.
//
// Let's assume that:
//   - charset = ['a', 'b']
//   - min length = 2, max length = 3
//   - total strings = 10
//
// Generated strings and counts:
//   - a   → 0 times
//   - aa  → 1 time
//   - aaa → 0.75 times
//   - aab → 0.75 times
//   - ab  → 1 time
//   - ...
//
// Precomputed values:
//   - countByLength = [0, 4, 6]
//   - countByPrefix = [0, 0, 1, 0.75]
//   - sumByPrefix   = [10, 5, 2.5, 0.75]
//
// Suppose we want to generate simpleString(7), let's trace the loop:
//   - remain -= countByPrefix[0] = 7 - 0 = 7
//     i = remain / sumByPrefix[1] = 7 / 5 = 1 (selects 'b')
//     remain -= sumByPrefix[1] * i = 7 - (5 * 1) = 2
//     prefix = ['b']
//   - remain -= countByPrefix[1] = 2 - 0 = 2
//     i = remain / sumByPrefix[2] = 2 / 2.5 = 0 (selects 'a')
//     remain -= sumByPrefix[2] * i = 2 - (2.5 * 0) = 2
//     prefix = ['b', 'a']
//   - remain -= countByPrefix[2] = 2 - 1 = 1
//     i = remain / sumByPrefix[3] = 1 / 0.75 = 1 (selects 'b')
//     remain -= sumByPrefix[3] * i = 1 - (0.75 * 1) = 0.25
//     prefix = ['b', 'a', 'b']
//   - remain -= countByPrefix[3] = 0.25 - 0.75 = -0.5
//     remain < 0 → break with result "bab"
func (g *StringGenerator) simpleString(number float64) string {
        prefix := make([]rune, 0, g.MaxLength)

        var prefixLen int

        for remain := number; ; {
                prefixLen = len(prefix)

                remain -= g.countByPrefix[prefixLen]
                if remain < 0 || g.sumByPrefix[prefixLen+1] == 0 {
                        break
                }

                i := int(remain / g.sumByPrefix[prefixLen+1])
                remain -= g.sumByPrefix[prefixLen+1] * float64(i)
                prefix = append(prefix, g.charset[i])
        }

        // The precision of float64 allows us to generate only 217 prefix characters (which is enough for us).
        // Within the ordered prefix, we can supplement with random characters.
        if prefixLen < g.MinLength {
                destLen := g.MinLength + int(number)%(g.MaxLength-g.MinLength+1)
                for i := range destLen - prefixLen {
                        prefix = append(prefix, g.charset[(int(number)+i*i)%len(g.charset)])
                }
        }

        return string(prefix)
}

// Value returns n-th string from range.
func (g *StringGenerator) Value(number float64, rowValues map[string]any) (any, error) {
        if g.Template != "" {
                val, err := g.templateString(number, rowValues)
                if err != nil {
                        return nil, errors.WithMessage(err, "failed to template string")
                }

                return val, nil
        }

        switch g.LogicalType {
        case models.FirstNameType:
                return g.firstName(number), nil
        case models.LastNameType:
                return g.lastName(number), nil
        case models.PhoneType:
                return g.phone(number), nil
        case models.TextType:
                return g.text(number)
        }

        return g.simpleString(number), nil
}

//nolint:cyclop
func (g *StringGenerator) ValuesCount(distinctValuesCountByColumn map[string]uint64) float64 {
        if g.Template != "" {
                return g.templateCardinality(distinctValuesCountByColumn)
        }

        switch g.LogicalType {
        case models.FirstNameType:
                return float64(len(g.localeModule.GetFirstNames(locale.AnyGender)))

        case models.LastNameType:
                return float64(len(g.localeModule.GetLastNames(locale.AnyGender)))

        case models.PhoneType:
                totalCount := float64(0)
                for _, pattern := range g.localeModule.GetPhonePatterns() {
                        totalCount += math.Pow(float64(10), float64(strings.Count(pattern, "#"))) //nolint:mnd
                }

                return totalCount

        case models.TextType:
                if g.MinLength > len(g.completions) {
                        return math.Inf(1)
                }

                totalCount := float64(0)
                for length := g.MinLength; length <= g.MaxLength && length+1 < len(g.completions); length++ {
                        totalCount += float64(g.completions[length+1])
                }

                return totalCount
        }

        totalCount := float64(0)
        for length := g.MinLength; length <= g.MaxLength; length++ {
                totalCount += math.Pow(float64(len(g.charset)), float64(length))
        }

        return totalCount
}

func (g *StringGenerator) templateCardinality(distinctValuesCountByColumn map[string]uint64) float64 {
        total := 1.0

        patternValMatches := rePatternVal.FindAllStringSubmatch(g.Template, -1)
        for _, match := range patternValMatches {
                pattern := match[1]
                if pattern == "" {
                        pattern = match[2]
                }

                total *= g.patternCardinality(pattern)
        }

        columns := common.ExtractValuesFromTemplate(g.Template)
        for _, column := range columns {
                if count, ok := distinctValuesCountByColumn[column]; ok && count > 0 {
                        total *= float64(count)
                }
        }

        return total
}

func (g *StringGenerator) patternCardinality(pattern string) float64 {
        total := 1.0

        if count := strings.Count(pattern, "A"); count > 0 {
                total *= math.Pow(float64(len(g.localeModule.LargeLetters())), float64(count))
        }

        if count := strings.Count(pattern, "a"); count > 0 {
                total *= math.Pow(float64(len(g.localeModule.SmallLetters())), float64(count))
        }

        if count := strings.Count(pattern, "0"); count > 0 {
                total *= math.Pow(float64(len(locale.Numbers)), float64(count))
        }

        if count := strings.Count(pattern, "#"); count > 0 {
                total *= math.Pow(float64(len(locale.SpecialChars)), float64(count))
        }

        return total
}

1	package value
2
3	import (
4	"math"
5	"math/big"
6	"regexp"
7	"slices"
8	"strings"
9
10	"github.com/flosch/pongo2"
11	"github.com/pkg/errors"
12	"github.com/tarantool/sdvg/internal/generator/common"
13	"github.com/tarantool/sdvg/internal/generator/models"
14	"github.com/tarantool/sdvg/internal/generator/usecase/general/locale"
15	"github.com/tarantool/sdvg/internal/generator/usecase/general/locale/en"
16	"github.com/tarantool/sdvg/internal/generator/usecase/general/locale/ru"
17	)
18
19	var (
20	rePatternVal = regexp.MustCompile(`pattern\((?:'([^'])'\|"([^"])")\)`)
21	)
22
23	// Verify interface compliance in compile time.
24	var _ Generator = (*StringGenerator)(nil)
25
26	// StringGenerator type is used to describe generator for strings.
27	type StringGenerator struct {
28	*models.ColumnStringParams
29	totalValuesCount uint64
30	localeModule locale.LocalModule
31	template *pongo2.Template
32	charset []rune
33	countByPrefix []float64
34	sumByPrefix []float64
35	completions []int64 // completions[i] stores the number of ways to form a text of length i
36	}
37
38	//nolint:cyclop
39	func (g *StringGenerator) Prepare() error {	2✔
40	if g.Template != "" {	3✔
41	template, err := pongo2.FromString(g.Template)	1✔
42	if err != nil {	1✔
NEW UNCOV 43	return errors.Errorf("failed to parse template: %s", err.Error())	×
NEW UNCOV 44	}	×
45
46	g.template = template	1✔
47	}
48
49	switch g.Locale {	2✔
50	case "ru":	1✔
51	g.localeModule = ru.NewLocaleModule(g.LogicalType, g.MinLength, g.MaxLength)	1✔
52	case "en":	2✔
53	g.localeModule = en.NewLocaleModule(g.LogicalType, g.MinLength, g.MaxLength)	2✔
54	default:	×
55	return errors.Errorf("unknown locale: %q", g.Locale)	×
56	}
57
58	switch g.LogicalType {	2✔
59	case models.FirstNameType:	1✔
60	if len(g.localeModule.GetFirstNames(locale.MaleGender)) == 0 {	1✔
61	return errors.Errorf("no male first names with length between %v and %v", g.MinLength, g.MaxLength)	×
62	}	×
63
64	if len(g.localeModule.GetFirstNames(locale.FemaleGender)) == 0 {	1✔
65	return errors.Errorf("no female first names with length between %v and %v", g.MinLength, g.MaxLength)	×
66	}	×
67	case models.LastNameType:	1✔
68	if len(g.localeModule.GetLastNames(locale.MaleGender)) == 0 {	1✔
UNCOV 69	return errors.Errorf("no male last names with length between %v and %v", g.MinLength, g.MaxLength)	×
UNCOV 70	}	×
71
72	if len(g.localeModule.GetLastNames(locale.FemaleGender)) == 0 {	1✔
73	return errors.Errorf("no female last names with length between %v and %v", g.MinLength, g.MaxLength)	×
UNCOV 74	}	×
75	case models.PhoneType:	1✔
76	if len(g.localeModule.GetPhonePatterns()) == 0 {	1✔
UNCOV 77	return errors.Errorf("no phone patterns with length between %v and %v", g.MinLength, g.MaxLength)	×
UNCOV 78	}	×
79	}
80
81	g.charset = make([]rune, 0)	2✔
82		2✔
83	if !g.WithoutLargeLetters {	4✔
84	g.charset = append(g.charset, g.localeModule.LargeLetters()...)	2✔
85	}	2✔
86
87	if !g.WithoutSmallLetters {	4✔
88	g.charset = append(g.charset, g.localeModule.SmallLetters()...)	2✔
89	}	2✔
90
91	if !g.WithoutNumbers {	4✔
92	g.charset = append(g.charset, locale.Numbers...)	2✔
93	}	2✔
94
95	if !g.WithoutSpecialChars {	4✔
96	g.charset = append(g.charset, locale.SpecialChars...)	2✔
97	}	2✔
98
99	slices.Sort(g.charset)	2✔
100		2✔
101	if g.LogicalType == models.TextType {	3✔
102	g.completions = g.calculateCompletions(g.MaxLength + 1)	1✔
103	}	1✔
104
105	return nil	2✔
106	}
107
108	func (g *StringGenerator) SetTotalCount(totalValuesCount uint64) error {	2✔
109	g.totalValuesCount = totalValuesCount	2✔
110		2✔
111	if g.LogicalType == "" && g.Template == "" {	4✔
112	countByLength := make([]float64, g.MaxLength+1)	2✔
113	avgRangeCount := math.Ceil(float64(totalValuesCount) / float64(g.MaxLength-g.MinLength+1))	2✔
114		2✔
115	for length := g.MinLength; length <= g.MaxLength; length++ {	4✔
116	rangeCount := math.Pow(float64(len(g.charset)), float64(length))	2✔
117		2✔
118	var currentLenCount float64	2✔
119	if avgRangeCount > rangeCount {	2✔
UNCOV 120	currentLenCount = rangeCount	×
UNCOV 121	avgRangeCount += (avgRangeCount - rangeCount) / float64(g.MaxLength-length)	×
122	} else {	2✔
123	currentLenCount = math.Ceil(avgRangeCount)	2✔
124	}	2✔
125
126	countByLength[length] = currentLenCount	2✔
127	}
128
129	g.countByPrefix = make([]float64, g.MaxLength+1)	2✔
130	g.sumByPrefix = make([]float64, g.MaxLength+1)	2✔
131		2✔
132	for prefix := 0; prefix <= g.MaxLength; prefix++ {	4✔
133	prefixDivider := math.Pow(float64(len(g.charset)), float64(prefix))	2✔
134	g.countByPrefix[prefix] = countByLength[prefix] / prefixDivider	2✔
135		2✔
136	for length := 0; length <= g.MaxLength-prefix; length++ {	4✔
137	g.sumByPrefix[prefix] += countByLength[length+prefix] / prefixDivider	2✔
138	}	2✔
139	}
140	}
141
142	return nil	2✔
143	}
144
145	// calculateCompletions precomputes completions.
146	func (g *StringGenerator) calculateCompletions(length int) []int64 {	1✔
147	words := g.localeModule.GetWords()	1✔
148	bytesPerChar := g.localeModule.GetBytesPerChar()	1✔
149	delimiterLen := len(locale.WordsDelimiter)	1✔
150		1✔
151	completionsBig := make([]*big.Int, length+1)	1✔
152	for i := range completionsBig {	2✔
153	completionsBig[i] = big.NewInt(0)	1✔
154	}	1✔
155
156	// Base case: one way to form a text of length 0 (the empty text).
157	completionsBig[0].SetInt64(1)	1✔
158		1✔
159	// Base case: all one-letter words.	1✔
160	for _, w := range words {	2✔
161	if len(w) == 1 {	2✔
162	completionsBig[1].Add(completionsBig[1], big.NewInt(1))	1✔
163	}	1✔
164	}
165
166	// For every target length, add ways by choosing each word that fits.
167	for l := 2; l <= length; l++ {	2✔
168	for _, w := range words {	2✔
169	wLen := len(w)/bytesPerChar + delimiterLen	1✔
170	if wLen <= l {	2✔
171	completionsBig[l].Add(completionsBig[l], completionsBig[l-wLen])	1✔
172	}	1✔
173	}
174	}
175
176	// convert from big.Int to int64
177	completions := make([]int64, 0, length+1)	1✔
178		1✔
179	for _, blockCount := range completionsBig {	2✔
180	if !blockCount.IsInt64() {	2✔
181	break	1✔
182	}
183
184	completions = append(completions, blockCount.Int64())	1✔
185	}
186
187	return completions	1✔
188	}
189
190	// templateString returns n-th string by template.
191	func (g *StringGenerator) templateString(number float64, rowValues map[string]any) (string, error) {	1✔
192	if rowValues == nil {	2✔
193	rowValues = make(map[string]any)	1✔
194	}	1✔
195
196	rowValues["pattern"] = func(pattern string) *pongo2.Value {	2✔
197	return pongo2.AsSafeValue(g.patternString(number, pattern))	1✔
198	}	1✔
199
200	val, err := g.template.Execute(rowValues)	1✔
201	if err != nil {	1✔
NEW UNCOV 202	return "", errors.New(err.Error())	×
NEW UNCOV 203	}	×
204
205	return val, nil	1✔
206	}
207
208	// patternString returns n-th string by pattern.
209	func (g *StringGenerator) patternString(number float64, pattern string) string {	1✔
210	val := []rune(pattern)	1✔
211	index := number / float64(g.totalValuesCount)	1✔
212		1✔
213	for i := range val {	2✔
214	var letters []rune	1✔
215		1✔
216	switch val[i] {	1✔
217	case 'A':	1✔
218	letters = g.localeModule.LargeLetters()	1✔
219	case 'a':	1✔
220	letters = g.localeModule.SmallLetters()	1✔
221	case '0':	1✔
222	letters = locale.Numbers	1✔
223	case '#':	1✔
224	letters = locale.SpecialChars	1✔
225	default:	1✔
226	continue	1✔
227	}
228
229	var pos int	1✔
230	pos, index = orderedPos(len(letters), index)	1✔
231	val[i] = letters[pos]	1✔
232	}
233
234	return string(val)	1✔
235	}
236
237	// firstName returns n-th first name from range.
238	func (g *StringGenerator) firstName(number float64) string {	1✔
239	firstNames := g.localeModule.GetFirstNames(locale.AnyGender)	1✔
240		1✔
241	pos := orderedInt64(0, int64(len(firstNames)-1), number, g.totalValuesCount)	1✔
242		1✔
243	return firstNames[pos]	1✔
244	}	1✔
245
246	// lastName returns n-th last name from range.
247	func (g *StringGenerator) lastName(number float64) string {	1✔
248	lastNames := g.localeModule.GetLastNames(locale.AnyGender)	1✔
249		1✔
250	pos := orderedInt64(0, int64(len(lastNames)-1), number, g.totalValuesCount)	1✔
251		1✔
252	return lastNames[pos]	1✔
253	}	1✔
254
255	// phone returns n-th phone number from range.
256	func (g *StringGenerator) phone(number float64) string {	1✔
257	patterns := g.localeModule.GetPhonePatterns()	1✔
258		1✔
259	pos := orderedInt64(0, int64(len(patterns)-1), number, g.totalValuesCount)	1✔
260		1✔
261	pattern := patterns[pos]	1✔
262	maxPhone := int64(math.Pow(10, float64(strings.Count(pattern, "#")))) - 1 //nolint:mnd	1✔
263		1✔
264	phone := orderedInt64(0, maxPhone, number, g.totalValuesCount)	1✔
265		1✔
266	return replaceWithNumber(pattern, '#', phone)	1✔
267	}	1✔
268
269	// text sorts texts only within their respective length groups.
270	// Texts of the same length will be ordered, but ordering
271	// between texts of different lengths is not guaranteed.
272	//
273	//nolint:cyclop
274	func (g *StringGenerator) text(num float64) (string, error) {	1✔
275	words := g.localeModule.GetWords()	1✔
276	oneLetterWords := g.localeModule.GetOneLetterWords()	1✔
277	oneLetterWordsLen := int64(len(oneLetterWords))	1✔
278		1✔
279	delimiter := locale.WordsDelimiter	1✔
280	delimiterLen := len(delimiter)	1✔
281		1✔
282	bytesPerChar := g.localeModule.GetBytesPerChar()	1✔
283		1✔
284	maxPreComputedLength := len(g.completions) - 1	1✔
285		1✔
286	wantedLen := g.MinLength + delimiterLen + int(num)%(g.MaxLength-g.MinLength+1)	1✔
287		1✔
288	number := int64(math.Floor(float64(g.completions[maxPreComputedLength]-1) * (num / float64(g.totalValuesCount))))	1✔
289		1✔
290	result := make([]byte, 0, wantedLen*bytesPerChar)	1✔
291		1✔
292	var textLen int	1✔
293		1✔
294	remaining := maxPreComputedLength	1✔
295	// Process until we've built the full text.	1✔
296	for remaining > 0 {	2✔
297	found := false	1✔
298	// Iterate over words in lexicographical order.	1✔
299	if remaining == 1 {	2✔
300	if number > oneLetterWordsLen-1 {	1✔
UNCOV 301	return "", errors.Errorf("remaining length is 1 but k: %v overflows: %v", number, oneLetterWordsLen)	×
UNCOV 302	}	×
303
304	result = append(result, oneLetterWords[number]...)	1✔
305		1✔
306	textLen++	1✔
307		1✔
308	break	1✔
309	}
310
311	for _, w := range words {	2✔
312	wLen := len(w)/bytesPerChar + delimiterLen	1✔
313	if wLen > remaining {	2✔
314	continue	1✔
315	}
316	// count = number of completions if we choose word w at this step.
317	count := g.completions[remaining-wLen]	1✔
318	// If k is within the block for word w, choose it.	1✔
319	if number < count {	2✔
320	result = append(result, w...)	1✔
321	result = append(result, delimiter...)	1✔
322		1✔
323	textLen += wLen	1✔
324		1✔
325	remaining -= wLen	1✔
326	found = true	1✔
327		1✔
328	break	1✔
329	}
330	// Otherwise, skip this block.
331	number -= count	1✔
332	}
333
334	if !found {	1✔
UNCOV 335	return "", errors.Errorf("index %v out of range for remaining length %d, %v", number, remaining, wantedLen)	×
UNCOV 336	}	×
337	}
338
339	for textLen < wantedLen {	2✔
340	w := words[number%int64(len(words)-1)]	1✔
341		1✔
342	result = append(result, w...)	1✔
343	result = append(result, delimiter...)	1✔
344		1✔
345	textLen += len(w)/bytesPerChar + delimiterLen	1✔
346	}	1✔
347
348	text := string(result)	1✔
349		1✔
350	if textLen > wantedLen {	2✔
351	if bytesPerChar == 1 {	2✔
352	text = text[:wantedLen]	1✔
353	} else {	2✔
354	text = string([]rune(text)[:wantedLen])	1✔
355	}	1✔
356	}
357
358	return text, nil	1✔
359	}
360
361	// simpleString generates a lexicographically ordered string based on the given number.
362	// The function ensures that strings of different lengths are evenly distributed.
363	//
364	// Prepared variables (from Prepare method):
365	// - countByLength - determines how many strings of each length should be generated; aims for an even distribution
366	// but adjusts when the number of possible strings at a given length is limited;
367	// - countByPrefix - determines how many times a given prefix should be repeated across generated strings;
368	// - sumByPrefix - keeps total number of strings that should be generated with a specific prefix of a certain length.
369	//
370	// Each iteration of loop follows these steps:
371	// - Subtracting the Current Prefix Group.
372	// countByPrefix[prefixLen] represents how many times the current prefix is repeated.
373	// We subtract this value from remain to determine if the target string falls within this group.
374	// If remain is negative, it means the desired index falls within the current prefix group, so we stop.
375	// If sumByPrefix[prefixLen+1] == 0, it means no further characters can be added, so we also stop.
376	// - Determining the Next Character.
377	// sumByPrefix[prefixLen+1] tells us how many strings exist for the next character choices.
378	// remain / sumByPrefix[prefixLen+1] determines how many prefixes we need to skip before choosing next character.
379	// We update remain according to reflect the choice. The selected character charset[i] is added to prefix.
380	//
381	// This approach ensures precision up to 217 characters in prefix length due to float64 limitations.
382	// Any additional characters required beyond the ordered prefix are filled in using a pattern based on `number`.
383	//
384	// Let's assume that:
385	// - charset = ['a', 'b']
386	// - min length = 2, max length = 3
387	// - total strings = 10
388	//
389	// Generated strings and counts:
390	// - a → 0 times
391	// - aa → 1 time
392	// - aaa → 0.75 times
393	// - aab → 0.75 times
394	// - ab → 1 time
395	// - ...
396	//
397	// Precomputed values:
398	// - countByLength = [0, 4, 6]
399	// - countByPrefix = [0, 0, 1, 0.75]
400	// - sumByPrefix = [10, 5, 2.5, 0.75]
401	//
402	// Suppose we want to generate simpleString(7), let's trace the loop:
403	// - remain -= countByPrefix[0] = 7 - 0 = 7
404	// i = remain / sumByPrefix[1] = 7 / 5 = 1 (selects 'b')
405	// remain -= sumByPrefix[1] * i = 7 - (5 * 1) = 2
406	// prefix = ['b']
407	// - remain -= countByPrefix[1] = 2 - 0 = 2
408	// i = remain / sumByPrefix[2] = 2 / 2.5 = 0 (selects 'a')
409	// remain -= sumByPrefix[2] * i = 2 - (2.5 * 0) = 2
410	// prefix = ['b', 'a']
411	// - remain -= countByPrefix[2] = 2 - 1 = 1
412	// i = remain / sumByPrefix[3] = 1 / 0.75 = 1 (selects 'b')
413	// remain -= sumByPrefix[3] * i = 1 - (0.75 * 1) = 0.25
414	// prefix = ['b', 'a', 'b']
415	// - remain -= countByPrefix[3] = 0.25 - 0.75 = -0.5
416	// remain < 0 → break with result "bab"
417	func (g *StringGenerator) simpleString(number float64) string {	2✔
418	prefix := make([]rune, 0, g.MaxLength)	2✔
419		2✔
420	var prefixLen int	2✔
421		2✔
422	for remain := number; ; {	4✔
423	prefixLen = len(prefix)	2✔
424		2✔
425	remain -= g.countByPrefix[prefixLen]	2✔
426	if remain < 0 \|\| g.sumByPrefix[prefixLen+1] == 0 {	4✔
427	break	2✔
428	}
429
430	i := int(remain / g.sumByPrefix[prefixLen+1])	2✔
431	remain -= g.sumByPrefix[prefixLen+1] * float64(i)	2✔
432	prefix = append(prefix, g.charset[i])	2✔
433	}
434
435	// The precision of float64 allows us to generate only 217 prefix characters (which is enough for us).
436	// Within the ordered prefix, we can supplement with random characters.
437	if prefixLen < g.MinLength {	2✔
UNCOV 438	destLen := g.MinLength + int(number)%(g.MaxLength-g.MinLength+1)	×
UNCOV 439	for i := range destLen - prefixLen {	×
UNCOV 440	prefix = append(prefix, g.charset[(int(number)+i*i)%len(g.charset)])	×
UNCOV 441	}	×
442	}
443
444	return string(prefix)	2✔
445	}
446
447	// Value returns n-th string from range.
448	func (g *StringGenerator) Value(number float64, rowValues map[string]any) (any, error) {	2✔
449	if g.Template != "" {	3✔
450	val, err := g.templateString(number, rowValues)	1✔
451	if err != nil {	1✔
NEW UNCOV 452	return nil, errors.WithMessage(err, "failed to template string")	×
NEW UNCOV 453	}	×
454
455	return val, nil	1✔
456	}
457
458	switch g.LogicalType {	2✔
459	case models.FirstNameType:	1✔
460	return g.firstName(number), nil	1✔
461	case models.LastNameType:	1✔
462	return g.lastName(number), nil	1✔
463	case models.PhoneType:	1✔
464	return g.phone(number), nil	1✔
465	case models.TextType:	1✔
466	return g.text(number)	1✔
467	}
468
469	return g.simpleString(number), nil	2✔
470	}
471
472	//nolint:cyclop
473	func (g *StringGenerator) ValuesCount(distinctValuesCountByColumn map[string]uint64) float64 {	2✔
474	if g.Template != "" {	3✔
475	return g.templateCardinality(distinctValuesCountByColumn)	1✔
476	}	1✔
477
478	switch g.LogicalType {	2✔
479	case models.FirstNameType:	1✔
480	return float64(len(g.localeModule.GetFirstNames(locale.AnyGender)))	1✔
481
482	case models.LastNameType:	1✔
483	return float64(len(g.localeModule.GetLastNames(locale.AnyGender)))	1✔
484
485	case models.PhoneType:	1✔
486	totalCount := float64(0)	1✔
487	for _, pattern := range g.localeModule.GetPhonePatterns() {	2✔
488	totalCount += math.Pow(float64(10), float64(strings.Count(pattern, "#"))) //nolint:mnd	1✔
489	}	1✔
490
491	return totalCount	1✔
492
493	case models.TextType:	1✔
494	if g.MinLength > len(g.completions) {	2✔
495	return math.Inf(1)	1✔
496	}	1✔
497
498	totalCount := float64(0)	1✔
499	for length := g.MinLength; length <= g.MaxLength && length+1 < len(g.completions); length++ {	2✔
500	totalCount += float64(g.completions[length+1])	1✔
501	}	1✔
502
503	return totalCount	1✔
504	}
505
506	totalCount := float64(0)	2✔
507	for length := g.MinLength; length <= g.MaxLength; length++ {	4✔
508	totalCount += math.Pow(float64(len(g.charset)), float64(length))	2✔
509	}	2✔
510
511	return totalCount	2✔
512	}
513
514	func (g *StringGenerator) templateCardinality(distinctValuesCountByColumn map[string]uint64) float64 {	1✔
515	total := 1.0	1✔
516		1✔
517	patternValMatches := rePatternVal.FindAllStringSubmatch(g.Template, -1)	1✔
518	for _, match := range patternValMatches {	2✔
519	pattern := match[1]	1✔
520	if pattern == "" {	1✔
NEW UNCOV 521	pattern = match[2]	×
NEW UNCOV 522	}	×
523
524	total *= g.patternCardinality(pattern)	1✔
525	}
526
527	columns := common.ExtractValuesFromTemplate(g.Template)	1✔
528	for _, column := range columns {	2✔
529	if count, ok := distinctValuesCountByColumn[column]; ok && count > 0 {	2✔
530	total *= float64(count)	1✔
531	}	1✔
532	}
533
534	return total	1✔
535	}
536
537	func (g *StringGenerator) patternCardinality(pattern string) float64 {	1✔
538	total := 1.0	1✔
539		1✔
540	if count := strings.Count(pattern, "A"); count > 0 {	2✔
541	total *= math.Pow(float64(len(g.localeModule.LargeLetters())), float64(count))	1✔
542	}	1✔
543
544	if count := strings.Count(pattern, "a"); count > 0 {	2✔
545	total *= math.Pow(float64(len(g.localeModule.SmallLetters())), float64(count))	1✔
546	}	1✔
547
548	if count := strings.Count(pattern, "0"); count > 0 {	2✔
549	total *= math.Pow(float64(len(locale.Numbers)), float64(count))	1✔
550	}	1✔
551
552	if count := strings.Count(pattern, "#"); count > 0 {	2✔
553	total *= math.Pow(float64(len(locale.SpecialChars)), float64(count))	1✔
554	}	1✔
555
556	return total	1✔
557	}

tarantool / sdvg / 16573271716

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous