16725361413

Committed 04 Aug 2025 02:03PM UTC coverage: 83.426%. First build

Build # 16725361413

Build Type

push

github

Committed by

web-flow

Commit Message

perf[duckdb]: add file footer caching (#4104)

Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>

---------

Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>

Run Details

188 of 206 new or added lines in 9 files covered. (91.26%)

46429 of 55653 relevant lines covered (83.43%)

450757.25 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.01

/vortex-duckdb/src/e2e_test/object_cache_test.rs

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

//! Test table function that demonstrates object cache usage

use std::ffi::CString;

use vortex::error::{VortexResult, vortex_err};

use crate::cpp::DUCKDB_TYPE;
use crate::duckdb::{
    BindInput, BindResult, ClientContext, DataChunk, LogicalType, TableFunction, TableInitInput,
};

#[derive(Debug, Clone)]
pub struct TestTableFunction;

#[derive(Debug, Clone)]
pub struct TestBindData {
    cache_key: String,
}

#[derive(Debug)]
pub struct TestGlobalState;

#[derive(Debug)]
pub struct TestLocalState;

#[derive(Debug, PartialEq)]
struct CachedData {
    message: String,
    count: i32,
}

impl TableFunction for TestTableFunction {
    type BindData = TestBindData;
    type GlobalState = TestGlobalState;
    type LocalState = TestLocalState;

    fn bind(
        client_context: &ClientContext,
        _input: &BindInput,
        result: &mut BindResult,
    ) -> VortexResult<Self::BindData> {
        let logical_type = LogicalType::new(DUCKDB_TYPE::DUCKDB_TYPE_BIGINT);
        result.add_result_column("test_value", &logical_type);

        let cache = client_context.object_cache();
        let cached_data = CachedData {
            message: "Hello from bind phase cache!".to_string(),
            count: 123,
        };

        // Store data in cache during bind
        cache.put("bind_phase_data", cached_data);

        Ok(TestBindData {
            cache_key: "test_table_function_data".to_string(),
        })
    }

    fn init_global(input: &TableInitInput<Self>) -> VortexResult<Self::GlobalState> {
        if let Ok(ctx) = input.client_context() {
            let cached_data = CachedData {
                message: "Hello from table function cache!".to_string(),
                count: 42,
            };
            let cache = ctx.object_cache();

            cache.put(&input.bind_data().cache_key, cached_data);
        }

        Ok(TestGlobalState)
    }

    fn init_local(
        _init: &TableInitInput<Self>,
        _global: &mut Self::GlobalState,
    ) -> VortexResult<Self::LocalState> {
        Ok(TestLocalState)
    }

    fn scan(
        _client_context: &ClientContext,
        _bind_data: &Self::BindData,
        _local_state: &mut Self::LocalState,
        _global_state: &mut Self::GlobalState,
        chunk: &mut DataChunk,
    ) -> VortexResult<()> {
        chunk.set_len(0);

        Ok(())
    }

    fn partition_data(
        _bind_data: &Self::BindData,
        _global_init_data: &mut Self::GlobalState,
        _local_init_data: &mut Self::LocalState,
    ) -> VortexResult<u64> {
        Ok(0)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::duckdb::Database;

    #[test]
    fn test_table_function_with_object_cache() -> VortexResult<()> {
        let db = Database::open_in_memory()?;
        let conn = db.connect()?;

        // Register our test table function
        let name =
            CString::new("test_cache_func").map_err(|e| vortex_err!("CString error: {}", e))?;
        conn.register_table_function::<TestTableFunction>(&name)?;

        // Call the table function - this should store data in the cache during init_global
        let _result = conn.query("SELECT * FROM test_cache_func()")?;

        // Try to verify that we can access the cached data from outside the table function
        // This part is optional since we're not sure if the object cache access is working yet
        let ctx = conn.client_context();
        if let Ok(ctx) = ctx {
            let cache = ctx.object_cache();
            // Check data from bind phase
            let bind_cached_data = cache.get::<CachedData>("bind_phase_data");
            if let Some(data) = bind_cached_data {
                assert_eq!(data.message, "Hello from bind phase cache!");
                assert_eq!(data.count, 123);
                println!("Successfully retrieved bind phase cached data!");
            }

            // Check data from init_global phase
            let cached_data = cache.get::<CachedData>("test_table_function_data");
            if let Some(data) = cached_data {
                assert_eq!(data.message, "Hello from table function cache!");
                assert_eq!(data.count, 42);
                println!("Successfully retrieved init_global phase cached data!");
            }
        }

        Ok(())
    }
}

1	// SPDX-License-Identifier: Apache-2.0
2	// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4	//! Test table function that demonstrates object cache usage
5
6	use std::ffi::CString;
7
8	use vortex::error::{VortexResult, vortex_err};
9
10	use crate::cpp::DUCKDB_TYPE;
11	use crate::duckdb::{
12	BindInput, BindResult, ClientContext, DataChunk, LogicalType, TableFunction, TableInitInput,
13	};
14
15	#[derive(Debug, Clone)]
16	pub struct TestTableFunction;
17
18	#[derive(Debug, Clone)]
19	pub struct TestBindData {
20	cache_key: String,
21	}
22
23	#[derive(Debug)]
24	pub struct TestGlobalState;
25
26	#[derive(Debug)]
27	pub struct TestLocalState;
28
29	#[derive(Debug, PartialEq)]
30	struct CachedData {
31	message: String,
32	count: i32,
33	}
34
35	impl TableFunction for TestTableFunction {
36	type BindData = TestBindData;
37	type GlobalState = TestGlobalState;
38	type LocalState = TestLocalState;
39
40	fn bind(	1✔
41	client_context: &ClientContext,	1✔
42	_input: &BindInput,	1✔
43	result: &mut BindResult,	1✔
44	) -> VortexResult<Self::BindData> {	1✔
45	let logical_type = LogicalType::new(DUCKDB_TYPE::DUCKDB_TYPE_BIGINT);	1✔
46	result.add_result_column("test_value", &logical_type);	1✔
47
48	let cache = client_context.object_cache();	1✔
49	let cached_data = CachedData {	1✔
50	message: "Hello from bind phase cache!".to_string(),	1✔
51	count: 123,	1✔
52	};	1✔
53
54	// Store data in cache during bind
55	cache.put("bind_phase_data", cached_data);	1✔
56
57	Ok(TestBindData {	1✔
58	cache_key: "test_table_function_data".to_string(),	1✔
59	})	1✔
60	}	1✔
61
62	fn init_global(input: &TableInitInput<Self>) -> VortexResult<Self::GlobalState> {	1✔
63	if let Ok(ctx) = input.client_context() {	1✔
64	let cached_data = CachedData {	1✔
65	message: "Hello from table function cache!".to_string(),	1✔
66	count: 42,	1✔
67	};	1✔
68	let cache = ctx.object_cache();	1✔
69		1✔
70	cache.put(&input.bind_data().cache_key, cached_data);	1✔
71	}	1✔
72
73	Ok(TestGlobalState)	1✔
74	}	1✔
75
76	fn init_local(	8✔
77	_init: &TableInitInput<Self>,	8✔
78	_global: &mut Self::GlobalState,	8✔
79	) -> VortexResult<Self::LocalState> {	8✔
80	Ok(TestLocalState)	8✔
81	}	8✔
82
83	fn scan(	8✔
84	_client_context: &ClientContext,	8✔
85	_bind_data: &Self::BindData,	8✔
86	_local_state: &mut Self::LocalState,	8✔
87	_global_state: &mut Self::GlobalState,	8✔
88	chunk: &mut DataChunk,	8✔
89	) -> VortexResult<()> {	8✔
90	chunk.set_len(0);	8✔
91
92	Ok(())	8✔
93	}	8✔
94
NEW 95	fn partition_data(	×
NEW 96	_bind_data: &Self::BindData,	×
NEW 97	_global_init_data: &mut Self::GlobalState,	×
NEW 98	_local_init_data: &mut Self::LocalState,	×
NEW 99	) -> VortexResult<u64> {	×
NEW 100	Ok(0)	×
NEW 101	}	×
102	}
103
104	#[cfg(test)]
105	mod tests {
106	use super::*;
107	use crate::duckdb::Database;
108
109	#[test]
110	fn test_table_function_with_object_cache() -> VortexResult<()> {	1✔
111	let db = Database::open_in_memory()?;	1✔
112	let conn = db.connect()?;	1✔
113
114	// Register our test table function
115	let name =	1✔
116	CString::new("test_cache_func").map_err(\|e\| vortex_err!("CString error: {}", e))?;	1✔
117	conn.register_table_function::<TestTableFunction>(&name)?;	1✔
118
119	// Call the table function - this should store data in the cache during init_global
120	let _result = conn.query("SELECT * FROM test_cache_func()")?;	1✔
121
122	// Try to verify that we can access the cached data from outside the table function
123	// This part is optional since we're not sure if the object cache access is working yet
124	let ctx = conn.client_context();	1✔
125	if let Ok(ctx) = ctx {	1✔
126	let cache = ctx.object_cache();	1✔
127	// Check data from bind phase
128	let bind_cached_data = cache.get::<CachedData>("bind_phase_data");	1✔
129	if let Some(data) = bind_cached_data {	1✔
130	assert_eq!(data.message, "Hello from bind phase cache!");	1✔
131	assert_eq!(data.count, 123);	1✔
132	println!("Successfully retrieved bind phase cached data!");	1✔
NEW 133	}	×
134
135	// Check data from init_global phase
136	let cached_data = cache.get::<CachedData>("test_table_function_data");	1✔
137	if let Some(data) = cached_data {	1✔
138	assert_eq!(data.message, "Hello from table function cache!");	1✔
139	assert_eq!(data.count, 42);	1✔
140	println!("Successfully retrieved init_global phase cached data!");	1✔
NEW 141	}	×
NEW 142	}	×
143
144	Ok(())	1✔
145	}	1✔
146	}

vortex-data / vortex / 16725361413

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous