brian/wanderer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improves initial meilisearch indexing speed

Browse Source
This commit is contained in:
Christian Beutel
2025-09-13 11:33:19 +02:00
parent 73d20b1cbc
commit b2be48a282
2 changed files with 109 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
db/main.go
View File

@@ -233,7 +233,7 @@ func createTrailHandler(client meilisearch.ServiceManager) func(e *core.RecordEv
if err != nil {
return err
}
if err := util.IndexTrail(e.App, record, author, client); err != nil {
if err := util.IndexTrails(e.App, []*core.Record{record}, client); err != nil {
return err
}
if !author.GetBool("isLocal") {
@@ -340,12 +340,7 @@ func createSummitLogHandler(client meilisearch.ServiceManager) func(e *core.Reco
return err
}
trailAuthor, err := e.App.FindFirstRecordByData("activitypub_actors", "id", trail.GetString("author"))
if err != nil {
return err
}
if err := util.IndexTrail(e.App, trail, trailAuthor, client); err != nil {
if err := util.IndexTrails(e.App, []*core.Record{trail}, client); err != nil {
return err
}
@@ -391,12 +386,7 @@ func deleteSummitLogHandler(client meilisearch.ServiceManager) func(e *core.Reco
return err
}
trailAuthor, err := e.App.FindFirstRecordByData("activitypub_actors", "id", trail.GetString("author"))
if err != nil {
return err
}
if err := util.IndexTrail(e.App, trail, trailAuthor, client); err != nil {
if err := util.IndexTrails(e.App, []*core.Record{trail}, client); err != nil {
return err
}
@@ -616,7 +606,7 @@ func createListHandler(client meilisearch.ServiceManager) func(e *core.RecordEve
return err
}
if err := util.IndexList(e.App, record, author, client); err != nil {
if err := util.IndexLists(e.App, []*core.Record{record}, client); err != nil {
return err
}
@@ -1309,7 +1299,7 @@ func bootstrapCategories(app core.App) error {
func bootstrapMeilisearchDocuments(app core.App, client meilisearch.ServiceManager) error {
// --- Trails ---
const pageSize = 100
const pageSize int64 = 100
var page int64 = 0
// Clear index before re-indexing
@@ -1321,7 +1311,7 @@ func bootstrapMeilisearchDocuments(app core.App, client meilisearch.ServiceManag
trails := []*core.Record{}
err := app.RecordQuery("trails").
Limit(pageSize).
Offset(page * int64(pageSize)).
Offset(page * pageSize).
All(&trails)
if err != nil {
return err
@@ -1330,70 +1320,9 @@ func bootstrapMeilisearchDocuments(app core.App, client meilisearch.ServiceManag
break
}
// Collect author IDs to fetch in batch
authorIDs := make([]string, 0, len(trails))
for _, t := range trails {
if id := t.GetString("author"); id != "" {
authorIDs = append(authorIDs, id)
}
}
// Fetch all authors in one go
authorArgs := make([]interface{}, len(authorIDs))
for i, v := range authorIDs {
authorArgs[i] = v
}
authors, err := app.FindAllRecords("activitypub_actors",
dbx.In("id", authorArgs...),
)
if err != nil {
return err
}
authorMap := make(map[string]*core.Record, len(authors))
for _, a := range authors {
authorMap[a.Id] = a
}
for _, trail := range trails {
author := authorMap[trail.GetString("author")]
if err := util.IndexTrail(app, trail, author, client); err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to index trail '%s': %v", trail.GetString("name"), err))
continue
}
// Shares
shares, err := app.FindAllRecords("trail_share",
dbx.NewExp("trail = {:trailId}", dbx.Params{"trailId": trail.Id}),
)
if err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to fetch shares for trail '%s': %v", trail.GetString("name"), err))
continue
}
shareActors := make([]string, len(shares))
for i, r := range shares {
shareActors[i] = r.GetString("actor")
}
if err := util.UpdateTrailShares(trail.Id, shareActors, client); err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to update trail shares '%s': %v", trail.GetString("name"), err))
}
// Likes
likes, err := app.FindAllRecords("trail_like",
dbx.NewExp("trail = {:trailId}", dbx.Params{"trailId": trail.Id}),
)
if err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to fetch likes for trail '%s': %v", trail.GetString("name"), err))
continue
}
likeActors := make([]string, len(likes))
for i, r := range likes {
likeActors[i] = r.GetString("actor")
}
if err := util.UpdateTrailLikes(trail.Id, likeActors, client); err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to update trail likes '%s': %v", trail.GetString("name"), err))
}
if err := util.IndexTrails(app, trails, client); err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to index trails page %d: %v", page, err))
continue
}
page++
@@ -1418,52 +1347,9 @@ func bootstrapMeilisearchDocuments(app core.App, client meilisearch.ServiceManag
break
}
// Collect author IDs in batch
authorIDs := make([]string, 0, len(lists))
for _, l := range lists {
if id := l.GetString("author"); id != "" {
authorIDs = append(authorIDs, id)
}
}
// Fetch all authors in one go
authorArgs := make([]interface{}, len(authorIDs))
for i, v := range authorIDs {
authorArgs[i] = v
}
authors, err := app.FindAllRecords("activitypub_actors",
dbx.In("id", authorArgs...),
)
if err != nil {
return err
}
authorMap := make(map[string]*core.Record, len(authors))
for _, a := range authors {
authorMap[a.Id] = a
}
for _, list := range lists {
author := authorMap[list.GetString("author")]
if err := util.IndexList(app, list, author, client); err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to index list '%s': %v", list.GetString("name"), err))
continue
}
shares, err := app.FindAllRecords("list_share",
dbx.NewExp("list = {:listId}", dbx.Params{"listId": list.Id}),
)
if err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to fetch list shares '%s': %v", list.GetString("name"), err))
continue
}
shareActors := make([]string, len(shares))
for i, r := range shares {
shareActors[i] = r.GetString("actor")
}
if err := util.UpdateListShares(list.Id, shareActors, client); err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to update list shares '%s': %v", list.GetString("name"), err))
}
if err := util.IndexLists(app, lists, client); err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to index list page %d: %v", page, err))
continue
}
page++

124
db/util/meilisearch.go
View File

@@ -88,9 +88,33 @@ func documentFromTrailRecord(app core.App, r *core.Record, author *core.Record,
}
if includeShares {
document["shares"] = []string{}
document["likes"] = []string{}
document["like_count"] = 0
trailShares := r.ExpandedAll("trail_share_via_trail")
if trailShares != nil {
sharedIDs := make([]string, len(trailShares))
for i, v := range trailShares {
sharedIDs[i] = v.GetString("actor")
}
document["shares"] = sharedIDs
} else {
document["shares"] = []string{}
}
trailLikes := r.ExpandedAll("trail_like_via_trail")
if trailLikes != nil {
likeIDs := make([]string, len(trailLikes))
for i, v := range trailLikes {
likeIDs[i] = v.GetString("actor")
}
document["likes"] = likeIDs
document["like_count"] = len(trailLikes)
} else {
document["likes"] = []string{}
document["like_count"] = 0
}
}
@@ -186,7 +210,7 @@ func documentFromListRecord(r *core.Record, author *core.Record, includeShares b
domain = author.GetString("domain")
}
document := map[string]interface{}{
document := map[string]any{
"id": r.Id,
"author": author.Id,
"author_name": author.GetString("preferred_username"),
@@ -206,7 +230,18 @@ func documentFromListRecord(r *core.Record, author *core.Record, includeShares b
}
if includeShares {
document["shares"] = []string{}
listShares := r.ExpandedAll("list_share_via_list")
if listShares != nil {
sharedIDs := make([]string, len(listShares))
for i, v := range listShares {
sharedIDs[i] = v.GetString("actor")
}
document["shares"] = sharedIDs
} else {
document["shares"] = []string{}
}
}
return document, nil
@@ -266,20 +301,40 @@ func documentFromRemoteRecord(r *core.Record, index string) (map[string]interfac
return document, nil
}
func IndexTrail(app core.App, r *core.Record, author *core.Record, client meilisearch.ServiceManager) error {
errs := app.ExpandRecord(r, []string{"tags"}, nil)
if len(errs) > 0 {
return fmt.Errorf("failed to expand tags: %v", errs)
func IndexTrails(app core.App, trails []*core.Record, client meilisearch.ServiceManager) error {
documents := make([]map[string]any, len(trails))
for i, r := range trails {
errs := app.ExpandRecord(r, []string{"tags"}, nil)
if len(errs) > 0 {
return fmt.Errorf("failed to expand tags: %v", errs)
}
errs = app.ExpandRecord(r, []string{"category"}, nil)
if len(errs) > 0 {
return fmt.Errorf("failed to expand category: %v", errs)
}
errs = app.ExpandRecord(r, []string{"trail_share_via_trail"}, nil)
if len(errs) > 0 {
return fmt.Errorf("failed to expand trail_share_via_trail: %v", errs)
}
errs = app.ExpandRecord(r, []string{"trail_like_via_trail"}, nil)
if len(errs) > 0 {
return fmt.Errorf("failed to expand trail_like_via_trail: %v", errs)
}
errs = app.ExpandRecord(r, []string{"author"}, nil)
if len(errs) > 0 {
return fmt.Errorf("failed to expand author: %v", errs)
}
author := r.ExpandedOne("author")
doc, err := documentFromTrailRecord(app, r, author, true)
if err != nil {
return err
}
documents[i] = doc
}
errs = app.ExpandRecord(r, []string{"category"}, nil)
if len(errs) > 0 {
return fmt.Errorf("failed to expand category: %v", errs)
}
doc, err := documentFromTrailRecord(app, r, author, true)
if err != nil {
return err
}
documents := []map[string]interface{}{doc}
if _, err := client.Index("trails").AddDocuments(documents); err != nil {
return err
@@ -338,17 +393,32 @@ func UpdateTrailLikes(trailId string, likes []string, client meilisearch.Service
return nil
}
func IndexList(app core.App, r *core.Record, author *core.Record, client meilisearch.ServiceManager) error {
errs := app.ExpandRecord(r, []string{"trails"}, nil)
if len(errs) > 0 {
return fmt.Errorf("failed to expand trails: %v", errs)
}
func IndexLists(app core.App, lists []*core.Record, client meilisearch.ServiceManager) error {
documents := make([]map[string]any, len(lists))
documents, err := documentFromListRecord(r, author, true)
if err != nil {
return err
for i, r := range lists {
errs := app.ExpandRecord(r, []string{"trails"}, nil)
if len(errs) > 0 {
return fmt.Errorf("failed to expand trails: %v", errs)
}
errs = app.ExpandRecord(r, []string{"list_share_via_list"}, nil)
if len(errs) > 0 {
return fmt.Errorf("failed to expand list_share_via_list: %v", errs)
}
errs = app.ExpandRecord(r, []string{"author"}, nil)
if len(errs) > 0 {
return fmt.Errorf("failed to expand author: %v", errs)
}
author := r.ExpandedOne("author")
doc, err := documentFromListRecord(r, author, true)
if err != nil {
return err
}
documents[i] = doc
}
if _, err = client.Index("lists").AddDocuments(documents); err != nil {
if _, err := client.Index("lists").AddDocuments(documents); err != nil {
return err
}