14810021233

Committed 03 May 2025 10:27AM UTC coverage: 41.832% (+2.7%) from 39.136%

Build # 14810021233

Build Type

push

github

Committed by

web-flow

Commit Message

configurable processing context for large files (#17)

Run Details

78 of 106 new or added lines in 7 files covered. (73.58%)

2 existing lines in 2 files now uncovered.

338 of 808 relevant lines covered (41.83%)

0.45 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

71.43

/internal/reader/reader.go

//
// Copyright (C) 2025 Dmitry Kolesnikov
//
// This file may be modified and distributed under the terms
// of the MIT license.  See the LICENSE file for details.
// https://github.com/fogfish/iq
//

package reader

import (
        "context"
        "io"
        "strings"

        spec "github.com/fogfish/iq/internal/prompt"
        "github.com/fogfish/scanner"
        "github.com/kshard/chatter"
        "github.com/spf13/viper"
)

const (
        STRATEGY_NONE      = "none"
        STRATEGY_SENTENCE  = "sentence"
        STRATEGY_PARAGRAPH = "paragraph"
        STRATEGY_CHUNK     = "chunk"
)

func New(kind string, chars string, size int, r io.Reader) scanner.Scanner {
        switch kind {
        case STRATEGY_SENTENCE:
                if len(chars) == 0 {
                        chars = scanner.EndOfSentence
                }
                return scanner.NewSentencer(chars, r)

        case STRATEGY_PARAGRAPH:
                if len(chars) == 0 {
                        chars = "\n\n"
                }
                return scanner.NewSlicer(chars, r)

        case STRATEGY_CHUNK:
                if len(chars) == 0 {
                        chars = scanner.EndOfSentence
                }
                if size == 0 {
                        size = 1024
                }
                return scanner.NewChunker(size, scanner.NewSentencer(chars, r))

        default:
                return scanner.NewIdentity(r)
        }
}

type Agent interface {
        PromptOnce(context.Context, *viper.Viper, ...chatter.Opt) ([]byte, error)
}

func Process(ctx context.Context, agt Agent, req *viper.Viper, r scanner.Scanner, w io.Writer) error {
        for r.Scan() {
                txt := strings.TrimSpace(r.Text())
                if len(txt) == 0 {
                        continue
                }

                req.Set(spec.YAML_BLOB, txt)
                reply, err := agt.PromptOnce(context.Background(), req)
                if err != nil {
                        return err
                }
                if _, err := w.Write(reply); err != nil {
                        return err
                }
                if _, err := w.Write([]byte("\n")); err != nil {
                        return err
                }
        }
        if err := r.Err(); err != nil {
                return err
        }

        return nil
}

1	//
2	// Copyright (C) 2025 Dmitry Kolesnikov
3	//
4	// This file may be modified and distributed under the terms
5	// of the MIT license. See the LICENSE file for details.
6	// https://github.com/fogfish/iq
7	//
8
9	package reader
10
11	import (
12	"context"
13	"io"
14	"strings"
15
16	spec "github.com/fogfish/iq/internal/prompt"
17	"github.com/fogfish/scanner"
18	"github.com/kshard/chatter"
19	"github.com/spf13/viper"
20	)
21
22	const (
23	STRATEGY_NONE = "none"
24	STRATEGY_SENTENCE = "sentence"
25	STRATEGY_PARAGRAPH = "paragraph"
26	STRATEGY_CHUNK = "chunk"
27	)
28
29	func New(kind string, chars string, size int, r io.Reader) scanner.Scanner {	1✔
30	switch kind {	1✔
31	case STRATEGY_SENTENCE:	1✔
32	if len(chars) == 0 {	2✔
33	chars = scanner.EndOfSentence	1✔
34	}	1✔
35	return scanner.NewSentencer(chars, r)	1✔
36
37	case STRATEGY_PARAGRAPH:	1✔
38	if len(chars) == 0 {	1✔
NEW 39	chars = "\n\n"	×
NEW 40	}	×
41	return scanner.NewSlicer(chars, r)	1✔
42
43	case STRATEGY_CHUNK:	1✔
44	if len(chars) == 0 {	2✔
45	chars = scanner.EndOfSentence	1✔
46	}	1✔
47	if size == 0 {	1✔
NEW 48	size = 1024	×
NEW 49	}	×
50	return scanner.NewChunker(size, scanner.NewSentencer(chars, r))	1✔
51
52	default:	1✔
53	return scanner.NewIdentity(r)	1✔
54	}
55	}
56
57	type Agent interface {
58	PromptOnce(context.Context, *viper.Viper, ...chatter.Opt) ([]byte, error)
59	}
60
61	func Process(ctx context.Context, agt Agent, req *viper.Viper, r scanner.Scanner, w io.Writer) error {	1✔
62	for r.Scan() {	2✔
63	txt := strings.TrimSpace(r.Text())	1✔
64	if len(txt) == 0 {	2✔
65	continue	1✔
66	}
67
68	req.Set(spec.YAML_BLOB, txt)	1✔
69	reply, err := agt.PromptOnce(context.Background(), req)	1✔
70	if err != nil {	1✔
NEW 71	return err	×
NEW 72	}	×
73	if _, err := w.Write(reply); err != nil {	1✔
NEW 74	return err	×
NEW 75	}	×
76	if _, err := w.Write([]byte("\n")); err != nil {	1✔
NEW 77	return err	×
NEW 78	}	×
79	}
80	if err := r.Err(); err != nil {	1✔
NEW 81	return err	×
NEW 82	}	×
83
84	return nil	1✔
85	}

fogfish / iq / 14810021233

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous