13269584591

Committed 11 Feb 2025 06:10PM UTC coverage: 82.268% (-0.3%) from 82.571%

Build # 13269584591

Build Type

push

github

Committed by

web-flow

Commit Message

Feat: support filters and transformation (#125)

feat: add new support for filters and transformation arguments in histogram endpoint

* fix: fix the naming for concept lists
...from ids to definitions (defs) and add some parsing for the new filter entries
in request body

* feat: working histogram endpoint POC for using tmp table for transformations

* feat: added first log transformation

* feat: added z-score transformation

* feat: add tempttablecache to include cleanup of oldest / least accessed item

* tmp: temporarily disable strict validation until frontend is fixed

* feat: add new ToSQL wrapper method

* feat: remove histogramConceptId from histogram endpoint
...and expect it as part of the variables instead

* fix: make sure only postgresql and sqlserver are accepted for now

* fix: naming improvements and extra unit tests

* fix: improve coverage of parsing.go

* feat: improve coverage of TempTableCache

* feat: improve coverage of TableExists

Coverage Stats

356 of 428 new or added lines in 10 files covered. (83.18%)

13 existing lines in 3 files now uncovered.

2343 of 2848 relevant lines covered (82.27%)

29.93 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

77.36

/utils/temptablecache.go

package utils

import (
        "sync"
)

// TempTableCache is a thread-safe cache for storing temporary table mappings
var TempTableCache = &CacheSimple{
        Data:        make(map[string]interface{}),
        MaxSize:     100, // Note: this might need some tweaking if in a highly concurrent space...which is not the case now. Setting it too high risks too many temp tables..filling db.
        accessOrder: make([]string, 0),
}

// Defines a thread-safe in-memory cache
type CacheSimple struct {
        Data        map[string]interface{}
        MaxSize     int
        mu          sync.RWMutex
        accessOrder []string // Keeps track of insertion order for cleanup
}

// Retrieves a value from the cache
func (c *CacheSimple) Get(key string) (interface{}, bool) {
        c.mu.RLock()
        defer c.mu.RUnlock()

        value, exists := c.Data[key]
        // If the key already exists, update its position in the queue, so it is kept a bit longer:
        if exists {
                c.moveToEnd(key)
        }
        return value, exists
}

// Adds a key-value pair to the cache
func (c *CacheSimple) Set(key string, value interface{}) {
        c.mu.Lock()
        defer c.mu.Unlock()

        // If the key already exists, update its value and move it to the end of the access order
        if _, exists := c.Data[key]; exists {
                c.Data[key] = value
                c.moveToEnd(key)
                return
        }

        // Add the new key-value pair
        c.Data[key] = value
        c.accessOrder = append(c.accessOrder, key)

        // If the cache exceeds its maximum size, remove the oldest / least recently accessed entry
        if len(c.accessOrder) > c.MaxSize {
                oldestKey := c.accessOrder[0]
                c.accessOrder = c.accessOrder[1:]
                delete(c.Data, oldestKey)
                c.removeFromOrder(oldestKey)
                // Optionally drop the corresponding table
                // dropTable(key) - probably not necessary if DB cleans up tmp tables automatically when session closes
        }
}

// Removes a key from the cache
func (c *CacheSimple) Delete(key string) {
        c.mu.Lock()
        defer c.mu.Unlock()

        if _, exists := c.Data[key]; exists {
                delete(c.Data, key)
                c.removeFromOrder(key)
                // Optionally drop the corresponding table
                // dropTable(key) - probably not necessary if DB cleans up tmp tables automatically when session closes
        }
}

// Moves a key to the end of the access order
func (c *CacheSimple) moveToEnd(key string) {
        for i, k := range c.accessOrder {
                if k == key {
                        c.accessOrder = append(c.accessOrder[:i], c.accessOrder[i+1:]...)
                        c.accessOrder = append(c.accessOrder, key)
                        break
                }
        }
}

// Removes a key from the access order
func (c *CacheSimple) removeFromOrder(key string) {
        for i, k := range c.accessOrder {
                if k == key {
                        c.accessOrder = append(c.accessOrder[:i], c.accessOrder[i+1:]...)
                        break
                }
        }
}

1	package utils
2
3	import (
4	"sync"
5	)
6
7	// TempTableCache is a thread-safe cache for storing temporary table mappings
8	var TempTableCache = &CacheSimple{
9	Data: make(map[string]interface{}),
10	MaxSize: 100, // Note: this might need some tweaking if in a highly concurrent space...which is not the case now. Setting it too high risks too many temp tables..filling db.
11	accessOrder: make([]string, 0),
12	}
13
14	// Defines a thread-safe in-memory cache
15	type CacheSimple struct {
16	Data map[string]interface{}
17	MaxSize int
18	mu sync.RWMutex
19	accessOrder []string // Keeps track of insertion order for cleanup
20	}
21
22	// Retrieves a value from the cache
23	func (c *CacheSimple) Get(key string) (interface{}, bool) {	20✔
24	c.mu.RLock()	20✔
25	defer c.mu.RUnlock()	20✔
26		20✔
27	value, exists := c.Data[key]	20✔
28	// If the key already exists, update its position in the queue, so it is kept a bit longer:	20✔
29	if exists {	28✔
30	c.moveToEnd(key)	8✔
31	}	8✔
32	return value, exists	20✔
33	}
34
35	// Adds a key-value pair to the cache
36	func (c *CacheSimple) Set(key string, value interface{}) {	24✔
37	c.mu.Lock()	24✔
38	defer c.mu.Unlock()	24✔
39		24✔
40	// If the key already exists, update its value and move it to the end of the access order	24✔
41	if _, exists := c.Data[key]; exists {	28✔
42	c.Data[key] = value	4✔
43	c.moveToEnd(key)	4✔
44	return	4✔
45	}	4✔
46
47	// Add the new key-value pair
48	c.Data[key] = value	20✔
49	c.accessOrder = append(c.accessOrder, key)	20✔
50		20✔
51	// If the cache exceeds its maximum size, remove the oldest / least recently accessed entry	20✔
52	if len(c.accessOrder) > c.MaxSize {	28✔
53	oldestKey := c.accessOrder[0]	8✔
54	c.accessOrder = c.accessOrder[1:]	8✔
55	delete(c.Data, oldestKey)	8✔
56	c.removeFromOrder(oldestKey)	8✔
57	// Optionally drop the corresponding table	8✔
58	// dropTable(key) - probably not necessary if DB cleans up tmp tables automatically when session closes	8✔
59	}	8✔
60	}
61
62	// Removes a key from the cache
NEW 63	func (c *CacheSimple) Delete(key string) {	×
NEW 64	c.mu.Lock()	×
NEW 65	defer c.mu.Unlock()	×
NEW 66		×
NEW 67	if _, exists := c.Data[key]; exists {	×
NEW 68	delete(c.Data, key)	×
NEW 69	c.removeFromOrder(key)	×
NEW 70	// Optionally drop the corresponding table	×
NEW 71	// dropTable(key) - probably not necessary if DB cleans up tmp tables automatically when session closes	×
NEW 72	}	×
73	}
74
75	// Moves a key to the end of the access order
76	func (c *CacheSimple) moveToEnd(key string) {	12✔
77	for i, k := range c.accessOrder {	39✔
78	if k == key {	39✔
79	c.accessOrder = append(c.accessOrder[:i], c.accessOrder[i+1:]...)	12✔
80	c.accessOrder = append(c.accessOrder, key)	12✔
81	break	12✔
82	}
83	}
84	}
85
86	// Removes a key from the access order
87	func (c *CacheSimple) removeFromOrder(key string) {	8✔
88	for i, k := range c.accessOrder {	32✔
89	if k == key {	24✔
NEW 90	c.accessOrder = append(c.accessOrder[:i], c.accessOrder[i+1:]...)	×
NEW 91	break	×
92	}
93	}
94	}

uc-cdis / cohort-middleware / 13269584591

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous