brian/wanderer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improves initial meilisearch indexing speed

Browse Source
This commit is contained in:
Christian Beutel
2025-09-13 11:33:19 +02:00
parent 73d20b1cbc
commit b2be48a282
2 changed files with 109 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
db/main.go
View File

@@ -233,7 +233,7 @@ func createTrailHandler(client meilisearch.ServiceManager) func(e *core.RecordEv
if err != nil {
return err
}
if err := util.IndexTrail(e.App, record, author, client); err != nil {
if err := util.IndexTrails(e.App, []*core.Record{record}, client); err != nil {
return err
}
if !author.GetBool("isLocal") {
@@ -340,12 +340,7 @@ func createSummitLogHandler(client meilisearch.ServiceManager) func(e *core.Reco
return err
}
trailAuthor, err := e.App.FindFirstRecordByData("activitypub_actors", "id", trail.GetString("author"))
if err != nil {
return err
}
if err := util.IndexTrail(e.App, trail, trailAuthor, client); err != nil {
if err := util.IndexTrails(e.App, []*core.Record{trail}, client); err != nil {
return err
}
@@ -391,12 +386,7 @@ func deleteSummitLogHandler(client meilisearch.ServiceManager) func(e *core.Reco
return err
}
trailAuthor, err := e.App.FindFirstRecordByData("activitypub_actors", "id", trail.GetString("author"))
if err != nil {
return err
}
if err := util.IndexTrail(e.App, trail, trailAuthor, client); err != nil {
if err := util.IndexTrails(e.App, []*core.Record{trail}, client); err != nil {
return err
}
@@ -616,7 +606,7 @@ func createListHandler(client meilisearch.ServiceManager) func(e *core.RecordEve
return err
}
if err := util.IndexList(e.App, record, author, client); err != nil {
if err := util.IndexLists(e.App, []*core.Record{record}, client); err != nil {
return err
}
@@ -1309,7 +1299,7 @@ func bootstrapCategories(app core.App) error {
func bootstrapMeilisearchDocuments(app core.App, client meilisearch.ServiceManager) error {
// --- Trails ---
const pageSize = 100
const pageSize int64 = 100
var page int64 = 0
// Clear index before re-indexing
@@ -1321,7 +1311,7 @@ func bootstrapMeilisearchDocuments(app core.App, client meilisearch.ServiceManag
trails := []*core.Record{}
err := app.RecordQuery("trails").
Limit(pageSize).
Offset(page * int64(pageSize)).
Offset(page * pageSize).
All(&trails)
if err != nil {
return err
@@ -1330,70 +1320,9 @@ func bootstrapMeilisearchDocuments(app core.App, client meilisearch.ServiceManag
break
}
// Collect author IDs to fetch in batch
authorIDs := make([]string, 0, len(trails))
for _, t := range trails {
if id := t.GetString("author"); id != "" {
authorIDs = append(authorIDs, id)
}
}
// Fetch all authors in one go
authorArgs := make([]interface{}, len(authorIDs))
for i, v := range authorIDs {
authorArgs[i] = v
}
authors, err := app.FindAllRecords("activitypub_actors",
dbx.In("id", authorArgs...),
)
if err != nil {
return err
}
authorMap := make(map[string]*core.Record, len(authors))
for _, a := range authors {
authorMap[a.Id] = a
}
for _, trail := range trails {
author := authorMap[trail.GetString("author")]
if err := util.IndexTrail(app, trail, author, client); err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to index trail '%s': %v", trail.GetString("name"), err))
continue
}
// Shares
shares, err := app.FindAllRecords("trail_share",
dbx.NewExp("trail = {:trailId}", dbx.Params{"trailId": trail.Id}),
)
if err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to fetch shares for trail '%s': %v", trail.GetString("name"), err))
continue
}
shareActors := make([]string, len(shares))
for i, r := range shares {
shareActors[i] = r.GetString("actor")
}
if err := util.UpdateTrailShares(trail.Id, shareActors, client); err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to update trail shares '%s': %v", trail.GetString("name"), err))
}
// Likes
likes, err := app.FindAllRecords("trail_like",
dbx.NewExp("trail = {:trailId}", dbx.Params{"trailId": trail.Id}),
)
if err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to fetch likes for trail '%s': %v", trail.GetString("name"), err))
continue
}
likeActors := make([]string, len(likes))
for i, r := range likes {
likeActors[i] = r.GetString("actor")
}
if err := util.UpdateTrailLikes(trail.Id, likeActors, client); err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to update trail likes '%s': %v", trail.GetString("name"), err))
}
if err := util.IndexTrails(app, trails, client); err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to index trails page %d: %v", page, err))
continue
}
page++
@@ -1418,52 +1347,9 @@ func bootstrapMeilisearchDocuments(app core.App, client meilisearch.ServiceManag
break
}
// Collect author IDs in batch
authorIDs := make([]string, 0, len(lists))
for _, l := range lists {
if id := l.GetString("author"); id != "" {
authorIDs = append(authorIDs, id)
}
}
// Fetch all authors in one go
authorArgs := make([]interface{}, len(authorIDs))
for i, v := range authorIDs {
authorArgs[i] = v
}
authors, err := app.FindAllRecords("activitypub_actors",
dbx.In("id", authorArgs...),
)
if err != nil {
return err
}
authorMap := make(map[string]*core.Record, len(authors))
for _, a := range authors {
authorMap[a.Id] = a
}
for _, list := range lists {
author := authorMap[list.GetString("author")]
if err := util.IndexList(app, list, author, client); err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to index list '%s': %v", list.GetString("name"), err))
continue
}
shares, err := app.FindAllRecords("list_share",
dbx.NewExp("list = {:listId}", dbx.Params{"listId": list.Id}),
)
if err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to fetch list shares '%s': %v", list.GetString("name"), err))
continue
}
shareActors := make([]string, len(shares))
for i, r := range shares {
shareActors[i] = r.GetString("actor")
}
if err := util.UpdateListShares(list.Id, shareActors, client); err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to update list shares '%s': %v", list.GetString("name"), err))
}
if err := util.IndexLists(app, lists, client); err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to index list page %d: %v", page, err))
continue
}
page++