git/watch-tool
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: implement new logic for local storage and exporters

Browse source
This commit is contained in:
Patryk Hegenberg 2025-09-23 14:14:53 +02:00
parent 491eeaabd7
commit 366aac9edc
11 changed files with 1170 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

26
log_processor.go
View file

@ -4,20 +4,16 @@ import (
"context"
"log/slog"
"time"
"github.com/elastic/go-elasticsearch/v8"
)
type LogProcessor struct {
sender ElasticsearchSender
baseIndex string
storage StorageInterface
batchSize int
}
func NewLogProcessor(es *elasticsearch.Client, baseIndex string) *LogProcessor {
func NewLogProcessor(storage StorageInterface) *LogProcessor {
return &LogProcessor{
sender: NewElasticsearchSender(es),
baseIndex: baseIndex,
storage: storage,
batchSize: 100,
}
}
@ -33,7 +29,7 @@ func (lp *LogProcessor) Start(ctx context.Context, logChan <-chan LogEntry) {
select {
case <-ctx.Done():
if len(batch) > 0 {
lp.sendBatch(batch)
lp.storeBatch(ctx, batch)
}
slog.Info("Log processor stopped")
return
@ -41,7 +37,7 @@ func (lp *LogProcessor) Start(ctx context.Context, logChan <-chan LogEntry) {
case entry, ok := <-logChan:
if !ok {
if len(batch) > 0 {
lp.sendBatch(batch)
lp.storeBatch(ctx, batch)
}
return
}
@ -49,30 +45,30 @@ func (lp *LogProcessor) Start(ctx context.Context, logChan <-chan LogEntry) {
batch = append(batch, entry)
if len(batch) >= lp.batchSize {
lp.sendBatch(batch)
lp.storeBatch(ctx, batch)
batch = batch[:0]
}
case <-ticker.C:
if len(batch) > 0 {
lp.sendBatch(batch)
lp.storeBatch(ctx, batch)
batch = batch[:0]
}
}
}
}
func (lp *LogProcessor) sendBatch(batch []LogEntry) {
func (lp *LogProcessor) storeBatch(ctx context.Context, batch []LogEntry) {
if len(batch) == 0 {
return
}
if err := lp.sender.SendBatch(lp.baseIndex, batch); err != nil {
slog.Error("error sending log batch", "error", err, "batch_size", len(batch))
if err := lp.storage.StoreBatch(ctx, batch); err != nil {
slog.Error("error storing log batch", "error", err, "batch_size", len(batch))
return
}
slog.Debug("Log batch sent successfully", "batch_size", len(batch))
slog.Debug("Log batch stored successfully", "batch_size", len(batch))
}
func (lp *LogProcessor) SetBatchSize(size int) {