13070263183

Committed 31 Jan 2025 09:45AM UTC coverage: 0.0%. Remained the same

Build # 13070263183

Build Type

Pull #5

github

Committed by

Civil

Commit Message

Refactor and improve bot

 * Fixed small bug in forward action
 * package statefulFilters renamed to chains
 * generic badgerdb interface
 * add filter that counts repeated messages
 * add filter that checks if message is a story
 * add filter that checks if message was marked as spam

Pull Request Pull Request #5: some improvements

Run Details

0 of 1422 new or added lines in 20 files covered. (0.0%)

21 existing lines in 3 files now uncovered.

0 of 3756 relevant lines covered (0.0%)

0.0 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/filters/chains/checkNevents/rule.go

package checkNevents

import (
        "github.com/dgraph-io/badger/v4"
        "github.com/mymmrac/telego"
        "go.uber.org/zap"
        "google.golang.org/protobuf/proto"
        "google.golang.org/protobuf/types/known/timestamppb"

        actions "github.com/Civil/tg-simple-regex-antispam/actions/interfaces"
        "github.com/Civil/tg-simple-regex-antispam/bannedDB"
        "github.com/Civil/tg-simple-regex-antispam/constants"
        "github.com/Civil/tg-simple-regex-antispam/filters/interfaces"
        "github.com/Civil/tg-simple-regex-antispam/filters/types/checkNeventsState"
        "github.com/Civil/tg-simple-regex-antispam/filters/types/scoringResult"
        badgerHelper "github.com/Civil/tg-simple-regex-antispam/helper/badger"
        "github.com/Civil/tg-simple-regex-antispam/helper/badger/badgerOpts"
        config2 "github.com/Civil/tg-simple-regex-antispam/helper/config"
        "github.com/Civil/tg-simple-regex-antispam/helper/tg"
)

type Filter struct {
        chainName      string
        n              int
        logger         *zap.Logger
        filteringRules []interfaces.FilteringRule

        bannedUsers bannedDB.BanDB

        stateDir string

        actions []actions.Action

        db *badger.DB

        isFinal                bool
        warnAboutAlreadyBanned bool

        tg.TGHaveAdminCommands
}

func New(logger *zap.Logger, chainName string, banDB bannedDB.BanDB, _ *telego.Bot, config map[string]any,
        filteringRules []interfaces.FilteringRule, actions []actions.Action,
) (interfaces.StatefulFilter, error) {
        stateDir, err := config2.GetOptionString(config, "state_dir")
        if err != nil {
                return nil, err
        }
        if stateDir == "" {
                return nil, constants.ErrStateDirEmpty
        }

        n, err := config2.GetOptionInt(config, "n")
        if err != nil {
                return nil, err
        }
        if n == 0 {
                return nil, constants.ErrNIsZero
        }

        isFinal, err := config2.GetOptionBoolWithDefault(config, "isFinal", false)
        if err != nil {
                return nil, err
        }

        warnAboutAlreadyBanned, err := config2.GetOptionBoolWithDefault(config, "warnAboutAlreadyBanned", false)
        if err != nil {
                return nil, err
        }

        badgerDB, err := badger.Open(badgerOpts.GetBadgerOptions(logger, chainName+"_DB", stateDir))
        if err != nil {
                return nil, err
        }

        f := &Filter{
                logger: logger.With(
                        zap.String("filter", chainName),
                        zap.String("filter_type", "checkNevents"),
                ),
                chainName:              chainName,
                stateDir:               stateDir,
                bannedUsers:            banDB,
                filteringRules:         filteringRules,
                actions:                actions,
                db:                     badgerDB,
                isFinal:                isFinal,
                n:                      n,
                warnAboutAlreadyBanned: warnAboutAlreadyBanned,
                TGHaveAdminCommands: tg.TGHaveAdminCommands{
                        Handlers: make(map[string]tg.AdminCMDHandlerFunc),
                },
        }

        for _, filter := range f.filteringRules {
                prefix := filter.TGAdminPrefix()
                if prefix != "" {
                        f.TGHaveAdminCommands.Handlers[prefix] = filter.HandleTGCommands
                }
        }

        f.TGHaveAdminCommands.Handlers[f.bannedUsers.TGAdminPrefix()] = f.bannedUsers.HandleTGCommands

        return f, nil
}

func Help() string {
        return "checkNevents requires `stateFile` parameter"
}

func (r *Filter) setState(userID int64, s *checkNeventsState.State) error {
        b, err := proto.Marshal(s)
        if err != nil {
                return err
        }
        return r.db.Update(
                func(txn *badger.Txn) error {
                        return txn.Set(badgerHelper.UserIDToKey(userID), b)
                })
}

func (r *Filter) getState(userID int64) (*checkNeventsState.State, error) {
        var s checkNeventsState.State
        err := r.db.View(
                func(txn *badger.Txn) error {
                        item, err := txn.Get(badgerHelper.UserIDToKey(userID))
                        if err != nil {
                                return err
                        }
                        return item.Value(func(val []byte) error {
                                return proto.Unmarshal(val, &s)
                        })
                })
        if err != nil {
                return nil, err
        }
        return &s, nil
}

func (r *Filter) RemoveState(userID int64) error {
        return r.db.Update(
                func(txn *badger.Txn) error {
                        return txn.Delete(badgerHelper.UserIDToKey(userID))
                })
}

func (r *Filter) Score(bot *telego.Bot, msg *telego.Message) *scoringResult.ScoringResult {
        r.logger.Debug("scoring message", zap.Any("message", msg))
        userID := msg.From.ID
        logger := r.logger.With(zap.Int64("userID", userID))
        maxScore := &scoringResult.ScoringResult{}
        if r.bannedUsers.IsBanned(userID) && r.warnAboutAlreadyBanned {
                logger.Warn("user is banned, but somehow sends messages, deleting them")
                maxScore.Score = 100
                maxScore.Reason = "user was already banned"
                err := r.applyActions(logger, maxScore, msg.Chat.ChatID(), msg, []int64{int64(msg.MessageID)}, userID)
                if err != nil {
                        logger.Error("failed to apply actions", zap.Error(err))
                }

                return maxScore
        }

        actualState, err := r.getState(userID)
        if err != nil {
                logger.Error("failed to get state", zap.Error(err))
                actualState = checkNeventsState.State_builder{
                        Verified:   proto.Bool(false),
                        MessageIds: make(map[int64]bool),
                        LastUpdate: timestamppb.Now(),
                }.Build()
        } else if actualState == nil || (!actualState.GetVerified() && len(actualState.GetMessageIds()) == 0) {
                logger.Debug("state is empty, creating an empty one",
                        zap.Error(err),
                )

                actualState = checkNeventsState.State_builder{
                        Verified:   proto.Bool(false),
                        MessageIds: make(map[int64]bool),
                        LastUpdate: timestamppb.Now(),
                }.Build()
        }

        // We already verified that user
        if actualState.GetVerified() {
                logger.Debug("user is not a spammer, already verified")
                return maxScore
        }

        messageIds := actualState.GetMessageIds()
        messageIds[int64(msg.MessageID)] = true
        actualState.SetMessageIds(messageIds)
        actualState.SetLastUpdate(timestamppb.Now())

        // Checking for the filters to match the message
        for _, filter := range r.filteringRules {
                score := filter.Score(bot, msg)
                if score.Score > maxScore.Score {
                        maxScore = score
                        if filter.IsFinal() {
                                break
                        }
                }
        }
        if maxScore.Score == 100 {
                // We don't care about State of a spammer, but we need to track if they are banned (at least for some time)
                logger.Debug("user is a spammer, banning them",
                        zap.String("username", msg.From.Username),
                )
                err = r.bannedUsers.BanUser(userID)
                if err != nil {
                        logger.Error("failed to ban user", zap.Error(err))
                        return maxScore
                }

                messageIds := make([]int64, 0, len(actualState.GetMessageIds()))
                for id := range actualState.GetMessageIds() {
                        messageIds = append(messageIds, id)
                }

                err = r.applyActions(logger, maxScore, msg.Chat.ChatID(), msg, messageIds, userID)
                if err != nil {
                        logger.Error("failed to apply actions", zap.Error(err))
                }

                err = r.RemoveState(userID)
                if err != nil {
                        logger.Error("failed to remove state", zap.Error(err))
                        return maxScore
                }
                return maxScore
        }
        logger.Debug("message verified, updating state")
        if len(actualState.GetMessageIds()) >= r.n {
                logger.Debug("reached threshold, marking user as verified", zap.Int("n", r.n))
                actualState.SetVerified(true)
                actualState.SetMessageIds(make(map[int64]bool))
        }
        err = r.setState(userID, actualState)
        if err != nil {
                logger.Error("failed to set new state",
                        zap.Any("new_state", actualState),
                        zap.Error(err),
                )
        }
        return maxScore
}

func (r *Filter) applyActions(logger *zap.Logger, score *scoringResult.ScoringResult, ChatID telego.ChatID, msg *telego.Message, messageIds []int64, userID int64) error {
        for _, action := range r.actions {
                var err error
                if action.PerMessage() {
                        err = action.ApplyToMessage(r, score, msg, nil)
                } else {
                        err = action.Apply(r, score, ChatID, messageIds, userID, nil)
                }

                if err != nil {
                        logger.Error("failed to apply action", zap.Any("action", action), zap.Error(err))
                }
        }
        return nil
}

func (r *Filter) IsStateful() bool {
        return true
}

func (r *Filter) GetFilterName() string {
        return r.chainName
}

func (r *Filter) GetName() string {
        return "checkNEvents"
}

func (r *Filter) IsFinal() bool {
        return r.isFinal
}

func (r *Filter) Close() error {
        return r.db.Close()
}

func (r *Filter) SaveState() error {
        return nil
}

func (r *Filter) LoadState() error {
        return nil
}

func (r *Filter) UnbanUser(userID int64) error {
        newState := checkNeventsState.State_builder{
                Verified:   proto.Bool(true),
                MessageIds: make(map[int64]bool),
                LastUpdate: timestamppb.Now(),
        }.Build()
        return r.setState(userID, newState)
}

func (r *Filter) TGAdminPrefix() string {
        return r.chainName
}

1	package checkNevents
2
3	import (
4	"github.com/dgraph-io/badger/v4"
5	"github.com/mymmrac/telego"
6	"go.uber.org/zap"
7	"google.golang.org/protobuf/proto"
8	"google.golang.org/protobuf/types/known/timestamppb"
9
10	actions "github.com/Civil/tg-simple-regex-antispam/actions/interfaces"
11	"github.com/Civil/tg-simple-regex-antispam/bannedDB"
12	"github.com/Civil/tg-simple-regex-antispam/constants"
13	"github.com/Civil/tg-simple-regex-antispam/filters/interfaces"
14	"github.com/Civil/tg-simple-regex-antispam/filters/types/checkNeventsState"
15	"github.com/Civil/tg-simple-regex-antispam/filters/types/scoringResult"
16	badgerHelper "github.com/Civil/tg-simple-regex-antispam/helper/badger"
17	"github.com/Civil/tg-simple-regex-antispam/helper/badger/badgerOpts"
18	config2 "github.com/Civil/tg-simple-regex-antispam/helper/config"
19	"github.com/Civil/tg-simple-regex-antispam/helper/tg"
20	)
21
22	type Filter struct {
23	chainName string
24	n int
25	logger *zap.Logger
26	filteringRules []interfaces.FilteringRule
27
28	bannedUsers bannedDB.BanDB
29
30	stateDir string
31
32	actions []actions.Action
33
34	db *badger.DB
35
36	isFinal bool
37	warnAboutAlreadyBanned bool
38
39	tg.TGHaveAdminCommands
40	}
41
42	func New(logger zap.Logger, chainName string, banDB bannedDB.BanDB, _ telego.Bot, config map[string]any,
43	filteringRules []interfaces.FilteringRule, actions []actions.Action,
44	) (interfaces.StatefulFilter, error) {	×
45	stateDir, err := config2.GetOptionString(config, "state_dir")	×
46	if err != nil {	×
47	return nil, err	×
48	}	×
49	if stateDir == "" {	×
NEW 50	return nil, constants.ErrStateDirEmpty	×
51	}	×
52
53	n, err := config2.GetOptionInt(config, "n")	×
54	if err != nil {	×
55	return nil, err	×
56	}	×
57	if n == 0 {	×
NEW 58	return nil, constants.ErrNIsZero	×
59	}	×
60
61	isFinal, err := config2.GetOptionBoolWithDefault(config, "isFinal", false)	×
62	if err != nil {	×
63	return nil, err	×
64	}	×
65
66	warnAboutAlreadyBanned, err := config2.GetOptionBoolWithDefault(config, "warnAboutAlreadyBanned", false)	×
67	if err != nil {	×
68	return nil, err	×
69	}	×
70
71	badgerDB, err := badger.Open(badgerOpts.GetBadgerOptions(logger, chainName+"_DB", stateDir))	×
72	if err != nil {	×
73	return nil, err	×
74	}	×
75
76	f := &Filter{	×
77	logger: logger.With(	×
78	zap.String("filter", chainName),	×
79	zap.String("filter_type", "checkNevents"),	×
80	),	×
81	chainName: chainName,	×
82	stateDir: stateDir,	×
83	bannedUsers: banDB,	×
84	filteringRules: filteringRules,	×
85	actions: actions,	×
86	db: badgerDB,	×
87	isFinal: isFinal,	×
88	n: n,	×
89	warnAboutAlreadyBanned: warnAboutAlreadyBanned,	×
90	TGHaveAdminCommands: tg.TGHaveAdminCommands{	×
91	Handlers: make(map[string]tg.AdminCMDHandlerFunc),	×
92	},	×
93	}	×
94		×
95	for _, filter := range f.filteringRules {	×
96	prefix := filter.TGAdminPrefix()	×
97	if prefix != "" {	×
98	f.TGHaveAdminCommands.Handlers[prefix] = filter.HandleTGCommands	×
99	}	×
100	}
101
102	f.TGHaveAdminCommands.Handlers[f.bannedUsers.TGAdminPrefix()] = f.bannedUsers.HandleTGCommands	×
103		×
104	return f, nil	×
105	}
106
107	func Help() string {	×
108	return "checkNevents requires `stateFile` parameter"	×
109	}	×
110
111	func (r Filter) setState(userID int64, s checkNeventsState.State) error {	×
112	b, err := proto.Marshal(s)	×
113	if err != nil {	×
114	return err	×
115	}	×
116	return r.db.Update(	×
117	func(txn *badger.Txn) error {	×
118	return txn.Set(badgerHelper.UserIDToKey(userID), b)	×
119	})	×
120	}
121
122	func (r Filter) getState(userID int64) (checkNeventsState.State, error) {	×
123	var s checkNeventsState.State	×
124	err := r.db.View(	×
125	func(txn *badger.Txn) error {	×
126	item, err := txn.Get(badgerHelper.UserIDToKey(userID))	×
127	if err != nil {	×
128	return err	×
129	}	×
130	return item.Value(func(val []byte) error {	×
131	return proto.Unmarshal(val, &s)	×
132	})	×
133	})
134	if err != nil {	×
135	return nil, err	×
136	}	×
137	return &s, nil	×
138	}
139
140	func (r *Filter) RemoveState(userID int64) error {	×
141	return r.db.Update(	×
142	func(txn *badger.Txn) error {	×
143	return txn.Delete(badgerHelper.UserIDToKey(userID))	×
144	})	×
145	}
146
147	func (r Filter) Score(bot telego.Bot, msg telego.Message) scoringResult.ScoringResult {	×
148	r.logger.Debug("scoring message", zap.Any("message", msg))	×
149	userID := msg.From.ID	×
150	logger := r.logger.With(zap.Int64("userID", userID))	×
151	maxScore := &scoringResult.ScoringResult{}	×
152	if r.bannedUsers.IsBanned(userID) && r.warnAboutAlreadyBanned {	×
153	logger.Warn("user is banned, but somehow sends messages, deleting them")	×
154	maxScore.Score = 100	×
155	maxScore.Reason = "user was already banned"	×
156	err := r.applyActions(logger, maxScore, msg.Chat.ChatID(), msg, []int64{int64(msg.MessageID)}, userID)	×
157	if err != nil {	×
158	logger.Error("failed to apply actions", zap.Error(err))	×
159	}	×
160
161	return maxScore	×
162	}
163
164	actualState, err := r.getState(userID)	×
165	if err != nil {	×
166	logger.Error("failed to get state", zap.Error(err))	×
NEW 167	actualState = checkNeventsState.State_builder{	×
NEW 168	Verified: proto.Bool(false),	×
169	MessageIds: make(map[int64]bool),	×
170	LastUpdate: timestamppb.Now(),	×
NEW 171	}.Build()	×
NEW 172	} else if actualState == nil \|\| (!actualState.GetVerified() && len(actualState.GetMessageIds()) == 0) {	×
173	logger.Debug("state is empty, creating an empty one",	×
174	zap.Error(err),	×
175	)	×
NEW 176		×
NEW 177	actualState = checkNeventsState.State_builder{	×
NEW 178	Verified: proto.Bool(false),	×
179	MessageIds: make(map[int64]bool),	×
180	LastUpdate: timestamppb.Now(),	×
NEW 181	}.Build()	×
182	}	×
183
184	// We already verified that user
NEW 185	if actualState.GetVerified() {	×
186	logger.Debug("user is not a spammer, already verified")	×
187	return maxScore	×
188	}	×
189
NEW 190	messageIds := actualState.GetMessageIds()	×
NEW 191	messageIds[int64(msg.MessageID)] = true	×
NEW 192	actualState.SetMessageIds(messageIds)	×
NEW 193	actualState.SetLastUpdate(timestamppb.Now())	×
194		×
195	// Checking for the filters to match the message	×
196	for _, filter := range r.filteringRules {	×
197	score := filter.Score(bot, msg)	×
198	if score.Score > maxScore.Score {	×
199	maxScore = score	×
200	if filter.IsFinal() {	×
201	break	×
202	}
203	}
204	}
205	if maxScore.Score == 100 {	×
206	// We don't care about State of a spammer, but we need to track if they are banned (at least for some time)	×
207	logger.Debug("user is a spammer, banning them",	×
208	zap.String("username", msg.From.Username),	×
209	)	×
210	err = r.bannedUsers.BanUser(userID)	×
211	if err != nil {	×
212	logger.Error("failed to ban user", zap.Error(err))	×
213	return maxScore	×
214	}	×
215
NEW 216	messageIds := make([]int64, 0, len(actualState.GetMessageIds()))	×
NEW 217	for id := range actualState.GetMessageIds() {	×
218	messageIds = append(messageIds, id)	×
219	}	×
220
221	err = r.applyActions(logger, maxScore, msg.Chat.ChatID(), msg, messageIds, userID)	×
222	if err != nil {	×
223	logger.Error("failed to apply actions", zap.Error(err))	×
224	}	×
225
226	err = r.RemoveState(userID)	×
227	if err != nil {	×
228	logger.Error("failed to remove state", zap.Error(err))	×
229	return maxScore	×
230	}	×
231	return maxScore	×
232	}
233	logger.Debug("message verified, updating state")	×
NEW 234	if len(actualState.GetMessageIds()) >= r.n {	×
235	logger.Debug("reached threshold, marking user as verified", zap.Int("n", r.n))	×
NEW 236	actualState.SetVerified(true)	×
NEW 237	actualState.SetMessageIds(make(map[int64]bool))	×
238	}	×
239	err = r.setState(userID, actualState)	×
240	if err != nil {	×
241	logger.Error("failed to set new state",	×
242	zap.Any("new_state", actualState),	×
243	zap.Error(err),	×
244	)	×
245	}	×
246	return maxScore	×
247	}
248
249	func (r Filter) applyActions(logger zap.Logger, score scoringResult.ScoringResult, ChatID telego.ChatID, msg telego.Message, messageIds []int64, userID int64) error {	×
250	for _, action := range r.actions {	×
251	var err error	×
252	if action.PerMessage() {	×
NEW 253	err = action.ApplyToMessage(r, score, msg, nil)	×
254	} else {	×
NEW 255	err = action.Apply(r, score, ChatID, messageIds, userID, nil)	×
256	}	×
257
258	if err != nil {	×
259	logger.Error("failed to apply action", zap.Any("action", action), zap.Error(err))	×
260	}	×
261	}
262	return nil	×
263	}
264
265	func (r *Filter) IsStateful() bool {	×
266	return true	×
267	}	×
268
269	func (r *Filter) GetFilterName() string {	×
270	return r.chainName	×
271	}	×
272
273	func (r *Filter) GetName() string {	×
274	return "checkNEvents"	×
275	}	×
276
277	func (r *Filter) IsFinal() bool {	×
278	return r.isFinal	×
279	}	×
280
281	func (r *Filter) Close() error {	×
282	return r.db.Close()	×
283	}	×
284
285	func (r *Filter) SaveState() error {	×
286	return nil	×
287	}	×
288
289	func (r *Filter) LoadState() error {	×
290	return nil	×
291	}	×
292
293	func (r *Filter) UnbanUser(userID int64) error {	×
NEW 294	newState := checkNeventsState.State_builder{	×
NEW 295	Verified: proto.Bool(true),	×
296	MessageIds: make(map[int64]bool),	×
297	LastUpdate: timestamppb.Now(),	×
NEW 298	}.Build()	×
299	return r.setState(userID, newState)	×
300	}	×
301
302	func (r *Filter) TGAdminPrefix() string {	×
303	return r.chainName	×
304	}	×

Civil / tg-simple-regex-antispam / 13070263183

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous