190cabe0f2
All checks were successful
check / check (push) Successful in 3m10s
The per-webhook archiveWriter registry in the database delivery target was never evicted, so a deleted webhook's writer -- and any archive file handle open within its debounce window -- lingered for the process lifetime. Separately, expiry pruning ran only when an archive was (re)opened, and reopens only happen on writes, so an archive belonging to a webhook that stopped receiving events kept its expired rows forever. Eviction: a new one-method delivery.WebhookEvictor interface (kept separate from Notifier: archiving lifecycle is not notification) is implemented by the Engine and injected into the handlers. Deleting a webhook, or deleting its last database target, drops the writer from the registry and closes its handle under the writer's own mutex, so eviction can never race an in-flight write. An evicted writer refuses further writes rather than reopening a file nothing holds. The archive file is deliberately left on disk: it is long-term storage an operator may want to keep or move away, and destroying it as a side effect of deleting a webhook would be unrecoverable. Idle sweep: a new ArchiveSweeper, modelled on the event RetentionReaper (fx lifecycle hooks, cancellable context, WaitGroup, ticker loop), prunes archives whose database target declares a positive expiry. It reuses the existing RETENTION_SWEEP_INTERVAL rather than adding a config key. It never creates an archive -- a missing file is skipped, and the reopen uses SQLite mode=rw so the file cannot be conjured even if it disappears mid-sweep -- routes the prune through the per-webhook writer so its mutex orders the sweep against concurrent writes, and leaves the archive closed so the move-the-file-away workflow keeps working. A failure for one webhook is logged and the sweep continues. Archives with no expiry or the expiry "never" are untouched.
426 lines
10 KiB
Go
426 lines
10 KiB
Go
package delivery_test
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"net/http"
|
|
"path/filepath"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"gorm.io/driver/sqlite"
|
|
"gorm.io/gorm"
|
|
"gorm.io/gorm/clause"
|
|
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
|
"sneak.berlin/go/webhooker/internal/database"
|
|
"sneak.berlin/go/webhooker/internal/delivery"
|
|
)
|
|
|
|
const (
|
|
// sweepRowOld and sweepRowNew are the event ids
|
|
// seedArchiveRows assigns to the first and second seeded
|
|
// rows.
|
|
sweepRowOld = "ev-0"
|
|
sweepRowNew = "ev-1"
|
|
|
|
// sweepConcurrentWrites is how many deliveries the
|
|
// concurrent write-plus-sweep test races against the sweep.
|
|
sweepConcurrentWrites = 20
|
|
)
|
|
|
|
// sweeperEnv bundles the pieces an archive sweep test drives:
|
|
// a main configuration database holding webhooks and targets, a
|
|
// delivery engine owning the archive writer registry, and the
|
|
// data directory the archive files live in.
|
|
type sweeperEnv struct {
|
|
sweeper *delivery.ArchiveSweeper
|
|
eng *delivery.Engine
|
|
mainDB *database.Database
|
|
dataDir string
|
|
}
|
|
|
|
func setupSweeperTest(t *testing.T) *sweeperEnv {
|
|
t.Helper()
|
|
|
|
dataDir := t.TempDir()
|
|
log := archiveTestLogger()
|
|
|
|
sqlDB, err := sql.Open(
|
|
"sqlite",
|
|
fmt.Sprintf(
|
|
"file:%s?mode=rwc",
|
|
filepath.Join(dataDir, "main.db"),
|
|
),
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
t.Cleanup(func() { _ = sqlDB.Close() })
|
|
|
|
gdb, err := gorm.Open(
|
|
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
mainDB := database.NewTestDatabase(gdb)
|
|
require.NoError(t, mainDB.Migrate())
|
|
|
|
eng := delivery.NewTestEngineWithDB(
|
|
mainDB,
|
|
database.NewTestWebhookDBManager(dataDir),
|
|
log,
|
|
&http.Client{Timeout: 5 * time.Second},
|
|
1,
|
|
)
|
|
|
|
return &sweeperEnv{
|
|
sweeper: delivery.NewTestArchiveSweeper(
|
|
mainDB, eng, log,
|
|
),
|
|
eng: eng,
|
|
mainDB: mainDB,
|
|
dataDir: dataDir,
|
|
}
|
|
}
|
|
|
|
// archivePath returns where the engine keeps a webhook's
|
|
// archive file.
|
|
func (env *sweeperEnv) archivePath(webhookID string) string {
|
|
return filepath.Join(
|
|
env.dataDir, fmt.Sprintf("archive-%s.db", webhookID),
|
|
)
|
|
}
|
|
|
|
// seedDatabaseTarget creates a webhook with one database target
|
|
// carrying the given target config JSON, and returns the
|
|
// webhook id.
|
|
func (env *sweeperEnv) seedDatabaseTarget(
|
|
t *testing.T, configJSON string,
|
|
) string {
|
|
t.Helper()
|
|
|
|
wh := &database.Webhook{
|
|
UserID: uuid.New().String(),
|
|
Name: "sweep-test",
|
|
}
|
|
require.NoError(
|
|
t,
|
|
env.mainDB.DB().
|
|
Omit(clause.Associations).
|
|
Create(wh).Error,
|
|
)
|
|
|
|
tgt := &database.Target{
|
|
WebhookID: wh.ID,
|
|
Name: "archive",
|
|
Type: database.TargetTypeDatabase,
|
|
Active: true,
|
|
Config: configJSON,
|
|
}
|
|
require.NoError(
|
|
t,
|
|
env.mainDB.DB().
|
|
Omit(clause.Associations).
|
|
Create(tgt).Error,
|
|
)
|
|
|
|
return wh.ID
|
|
}
|
|
|
|
// seedArchiveRows creates the archive file for a webhook and
|
|
// inserts one row per supplied archived-at timestamp, returning
|
|
// the archive path. The handle is closed before returning, so
|
|
// the archive is idle exactly as it would be with no traffic.
|
|
func (env *sweeperEnv) seedArchiveRows(
|
|
t *testing.T, webhookID string, archivedAt ...time.Time,
|
|
) string {
|
|
t.Helper()
|
|
|
|
path := env.archivePath(webhookID)
|
|
|
|
sqlDB, err := sql.Open(
|
|
"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
gdb, err := gorm.Open(
|
|
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
require.NoError(
|
|
t, gdb.AutoMigrate(&delivery.ExportArchivedEvent{}),
|
|
)
|
|
|
|
for i, at := range archivedAt {
|
|
row := delivery.ExportArchivedEvent{
|
|
EventID: fmt.Sprintf("ev-%d", i),
|
|
WebhookID: webhookID,
|
|
Method: http.MethodPost,
|
|
Body: `{"seeded":true}`,
|
|
ArchivedAt: at,
|
|
}
|
|
require.NoError(t, gdb.Create(&row).Error)
|
|
}
|
|
|
|
require.NoError(t, sqlDB.Close())
|
|
|
|
return path
|
|
}
|
|
|
|
// archivedEventIDs returns the event ids currently stored in an
|
|
// archive file, read through a separate read-only handle.
|
|
func archivedEventIDs(
|
|
t *testing.T, path string,
|
|
) []string {
|
|
t.Helper()
|
|
|
|
var rows []delivery.ExportArchivedEvent
|
|
|
|
rdb := openArchiveDBForRead(t, path)
|
|
require.NoError(t, rdb.Order("event_id").Find(&rows).Error)
|
|
|
|
ids := make([]string, 0, len(rows))
|
|
for i := range rows {
|
|
ids = append(ids, rows[i].EventID)
|
|
}
|
|
|
|
return ids
|
|
}
|
|
|
|
// TestArchiveSweep_PrunesIdleArchive is the core regression
|
|
// test for this issue: an archive that receives no further
|
|
// writes must still lose its expired rows. Before the sweeper
|
|
// existed, pruning only ever ran on a write-triggered reopen,
|
|
// so an idle archive kept expired rows forever.
|
|
func TestArchiveSweep_PrunesIdleArchive(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
env := setupSweeperTest(t)
|
|
|
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
|
|
|
now := time.Now()
|
|
path := env.seedArchiveRows(
|
|
t, webhookID,
|
|
now.Add(-48*time.Hour),
|
|
now.Add(-time.Minute),
|
|
)
|
|
|
|
require.Equal(
|
|
t, []string{sweepRowOld, sweepRowNew},
|
|
archivedEventIDs(t, path),
|
|
)
|
|
|
|
env.sweeper.ExportSweep(context.Background())
|
|
|
|
assert.Equal(
|
|
t, []string{sweepRowNew}, archivedEventIDs(t, path),
|
|
"the sweep should prune rows older than the expiry "+
|
|
"from an idle archive and keep the rest",
|
|
)
|
|
}
|
|
|
|
// TestArchiveSweep_LeavesArchiveClosed proves the sweep does
|
|
// not hold the archive open afterwards, so an operator can
|
|
// still move the file away for offline retention.
|
|
func TestArchiveSweep_LeavesArchiveClosed(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
env := setupSweeperTest(t)
|
|
|
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
|
env.seedArchiveRows(
|
|
t, webhookID, time.Now().Add(-48*time.Hour),
|
|
)
|
|
|
|
env.sweeper.ExportSweep(context.Background())
|
|
|
|
assert.False(
|
|
t, env.eng.ExportArchiveHandleOpen(webhookID),
|
|
"an idle archive must end the sweep closed",
|
|
)
|
|
}
|
|
|
|
// TestArchiveSweep_NeverExpiryUntouched proves the sweep is a
|
|
// no-op for the default retention policy, so archives with no
|
|
// expiry (or the literal "never") behave exactly as before.
|
|
func TestArchiveSweep_NeverExpiryUntouched(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
for _, configJSON := range []string{
|
|
`{"expiry":"never"}`,
|
|
`{"expiry":""}`,
|
|
"",
|
|
} {
|
|
env := setupSweeperTest(t)
|
|
|
|
webhookID := env.seedDatabaseTarget(t, configJSON)
|
|
path := env.seedArchiveRows(
|
|
t, webhookID,
|
|
time.Now().Add(-10000*time.Hour),
|
|
)
|
|
|
|
env.sweeper.ExportSweep(context.Background())
|
|
|
|
assert.Equal(
|
|
t, []string{sweepRowOld}, archivedEventIDs(t, path),
|
|
"config %q must keep rows forever", configJSON,
|
|
)
|
|
}
|
|
}
|
|
|
|
// TestArchiveSweep_DoesNotCreateArchiveFile proves the sweep
|
|
// never conjures an archive: a webhook with a database target
|
|
// that has never received an event must still have no archive
|
|
// file (nor SQLite sidecar) after a sweep.
|
|
func TestArchiveSweep_DoesNotCreateArchiveFile(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
env := setupSweeperTest(t)
|
|
|
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
|
path := env.archivePath(webhookID)
|
|
|
|
require.NoFileExists(t, path)
|
|
|
|
env.sweeper.ExportSweep(context.Background())
|
|
|
|
for _, suffix := range []string{"", "-wal", "-shm"} {
|
|
assert.NoFileExists(
|
|
t, path+suffix,
|
|
"the sweep must not create an archive file",
|
|
)
|
|
}
|
|
}
|
|
|
|
// TestArchiveSweep_DoesNotCreateAfterWriterExists covers the
|
|
// same guarantee once a writer is cached in the registry but
|
|
// the file itself is still absent (for instance because the
|
|
// operator moved the archive away).
|
|
func TestArchiveSweep_DoesNotCreateAfterWriterExists(
|
|
t *testing.T,
|
|
) {
|
|
t.Parallel()
|
|
|
|
env := setupSweeperTest(t)
|
|
|
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
|
|
|
path, err := env.eng.ExportEnsureArchiveWriter(webhookID)
|
|
require.NoError(t, err)
|
|
require.NoFileExists(t, path)
|
|
|
|
env.sweeper.ExportSweep(context.Background())
|
|
|
|
assert.NoFileExists(t, path)
|
|
}
|
|
|
|
// TestArchiveSweep_SkipsDeletedWebhookTargets proves that the
|
|
// sweep ignores targets soft-deleted along with their webhook,
|
|
// so a deleted webhook's archive is never reopened.
|
|
func TestArchiveSweep_SkipsDeletedWebhookTargets(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
env := setupSweeperTest(t)
|
|
|
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
|
path := env.seedArchiveRows(
|
|
t, webhookID, time.Now().Add(-48*time.Hour),
|
|
)
|
|
|
|
require.NoError(
|
|
t,
|
|
env.mainDB.DB().
|
|
Where("webhook_id = ?", webhookID).
|
|
Delete(&database.Target{}).Error,
|
|
)
|
|
|
|
env.sweeper.ExportSweep(context.Background())
|
|
|
|
assert.Equal(
|
|
t, []string{sweepRowOld}, archivedEventIDs(t, path),
|
|
"a deleted target's archive must be left alone",
|
|
)
|
|
}
|
|
|
|
// TestArchiveSweep_ConcurrentWrites proves the sweep serialises
|
|
// against writes through the per-webhook writer mutex. Run
|
|
// under -race, an unsynchronised sweep would be caught here.
|
|
func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
env := setupSweeperTest(t)
|
|
|
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
|
|
|
webhookDB := testWebhookDB(t)
|
|
|
|
// The deliveries are seeded up front, on the test's own
|
|
// goroutine: the seed helpers assert, and testify assertions
|
|
// must not run off the test goroutine.
|
|
deliveries := make(
|
|
[]*database.Delivery, 0, sweepConcurrentWrites,
|
|
)
|
|
|
|
for range sweepConcurrentWrites {
|
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
|
event.WebhookID = webhookID
|
|
|
|
deliveries = append(
|
|
deliveries,
|
|
seedDatabaseTargetDelivery(
|
|
t, webhookDB, event, `{"expiry":"1h"}`,
|
|
),
|
|
)
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
wg.Add(2)
|
|
|
|
go func() {
|
|
defer wg.Done()
|
|
|
|
for _, d := range deliveries {
|
|
env.eng.ExportDeliverDatabase(webhookDB, d)
|
|
}
|
|
}()
|
|
|
|
go func() {
|
|
defer wg.Done()
|
|
|
|
for range sweepConcurrentWrites {
|
|
env.sweeper.ExportSweep(context.Background())
|
|
}
|
|
}()
|
|
|
|
wg.Wait()
|
|
|
|
assert.FileExists(t, env.archivePath(webhookID))
|
|
}
|
|
|
|
// TestArchiveSweeper_StopsCleanly proves the background loop
|
|
// exits on OnStop rather than leaking a goroutine.
|
|
func TestArchiveSweeper_StopsCleanly(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
env := setupSweeperTest(t)
|
|
|
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
|
env.seedArchiveRows(
|
|
t, webhookID, time.Now().Add(-48*time.Hour),
|
|
)
|
|
|
|
env.sweeper.ExportSetInterval(time.Millisecond)
|
|
env.sweeper.ExportStart(context.Background())
|
|
|
|
// stop blocks on the loop's WaitGroup, so returning at all
|
|
// proves the loop observed the cancellation and exited.
|
|
env.sweeper.ExportStop()
|
|
}
|