13098753112

Committed 02 Feb 2025 12:26PM UTC coverage: 3.411% (+3.4%) from 0.0%

Build # 13098753112

Build Type

push

github

Committed by

web-flow

Commit Message

LLM query/reply caching module (#13)

Run Details

25 of 35 new or added lines in 1 file covered. (71.43%)

25 of 733 relevant lines covered (3.41%)

0.04 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

71.43

/cache/cache.go

//
// Copyright (C) 2025 Dmitry Kolesnikov
//
// This file may be modified and distributed under the terms
// of the MIT license.  See the LICENSE file for details.
// https://github.com/kshard/chatter
//

package cache

import (
        "context"
        "crypto/sha1"
        "encoding"
        "log/slog"

        "github.com/kshard/chatter"
)

// Creates caching layer for LLM client.
//
// Use github.com/akrylysov/pogreb to cache chatter on local file systems:
//
//        llm, err := /* create LLM client */
//        db, err := pogreb.Open("llm.cache", nil)
//        text := cache.New(db, llm)
func New(cache Cache, chatter chatter.Chatter) *Client {
        return &Client{
                Chatter: chatter,
                cache:   cache,
        }
}

func (c *Client) HashKey(prompt encoding.TextMarshaler) ([]byte, error) {
        b, err := prompt.MarshalText()
        if err != nil {
                return nil, err
        }

        hash := sha1.New()
        hash.Write(b)
        return hash.Sum(nil), nil
}

func (c *Client) Prompt(ctx context.Context, prompt encoding.TextMarshaler, opts ...func(*chatter.Options)) (string, error) {
        hkey, err := c.HashKey(prompt)
        if err != nil {
                return "", err
        }

        val, err := c.cache.Get(hkey)
        if err != nil {
                return "", err
        }

        if len(val) != 0 {
                return string(val), nil
        }

        reply, err := c.Chatter.Prompt(ctx, prompt, opts...)
        if err != nil {
                return "", err
        }

        err = c.cache.Put(hkey, []byte(reply))
        if err != nil {
                slog.Warn("failed to cache LLM reply", "err", err)
        }

        return reply, nil
}

1	//
2	// Copyright (C) 2025 Dmitry Kolesnikov
3	//
4	// This file may be modified and distributed under the terms
5	// of the MIT license. See the LICENSE file for details.
6	// https://github.com/kshard/chatter
7	//
8
9	package cache
10
11	import (
12	"context"
13	"crypto/sha1"
14	"encoding"
15	"log/slog"
16
17	"github.com/kshard/chatter"
18	)
19
20	// Creates caching layer for LLM client.
21	//
22	// Use github.com/akrylysov/pogreb to cache chatter on local file systems:
23	//
24	// llm, err := /* create LLM client */
25	// db, err := pogreb.Open("llm.cache", nil)
26	// text := cache.New(db, llm)
27	func New(cache Cache, chatter chatter.Chatter) *Client {	1✔
28	return &Client{	1✔
29	Chatter: chatter,	1✔
30	cache: cache,	1✔
31	}	1✔
32	}	1✔
33
34	func (c *Client) HashKey(prompt encoding.TextMarshaler) ([]byte, error) {	1✔
35	b, err := prompt.MarshalText()	1✔
36	if err != nil {	1✔
NEW 37	return nil, err	×
NEW 38	}	×
39
40	hash := sha1.New()	1✔
41	hash.Write(b)	1✔
42	return hash.Sum(nil), nil	1✔
43	}
44
45	func (c Client) Prompt(ctx context.Context, prompt encoding.TextMarshaler, opts ...func(chatter.Options)) (string, error) {	1✔
46	hkey, err := c.HashKey(prompt)	1✔
47	if err != nil {	1✔
NEW 48	return "", err	×
NEW 49	}	×
50
51	val, err := c.cache.Get(hkey)	1✔
52	if err != nil {	1✔
NEW 53	return "", err	×
NEW 54	}	×
55
56	if len(val) != 0 {	2✔
57	return string(val), nil	1✔
58	}	1✔
59
60	reply, err := c.Chatter.Prompt(ctx, prompt, opts...)	1✔
61	if err != nil {	1✔
NEW 62	return "", err	×
NEW 63	}	×
64
65	err = c.cache.Put(hkey, []byte(reply))	1✔
66	if err != nil {	1✔
NEW 67	slog.Warn("failed to cache LLM reply", "err", err)	×
NEW 68	}	×
69
70	return reply, nil	1✔
71	}

kshard / chatter / 13098753112

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous