From b2be48a282f190901d6441fd80c8b6c18db60262 Mon Sep 17 00:00:00 2001 From: Christian Beutel <> Date: Sat, 13 Sep 2025 11:33:19 +0200 Subject: [PATCH] improves initial meilisearch indexing speed --- db/main.go | 138 ++++------------------------------------- db/util/meilisearch.go | 124 ++++++++++++++++++++++++++++-------- 2 files changed, 109 insertions(+), 153 deletions(-) diff --git a/db/main.go b/db/main.go index 6758ac50..31184b4b 100644 --- a/db/main.go +++ b/db/main.go @@ -233,7 +233,7 @@ func createTrailHandler(client meilisearch.ServiceManager) func(e *core.RecordEv if err != nil { return err } - if err := util.IndexTrail(e.App, record, author, client); err != nil { + if err := util.IndexTrails(e.App, []*core.Record{record}, client); err != nil { return err } if !author.GetBool("isLocal") { @@ -340,12 +340,7 @@ func createSummitLogHandler(client meilisearch.ServiceManager) func(e *core.Reco return err } - trailAuthor, err := e.App.FindFirstRecordByData("activitypub_actors", "id", trail.GetString("author")) - if err != nil { - return err - } - - if err := util.IndexTrail(e.App, trail, trailAuthor, client); err != nil { + if err := util.IndexTrails(e.App, []*core.Record{trail}, client); err != nil { return err } @@ -391,12 +386,7 @@ func deleteSummitLogHandler(client meilisearch.ServiceManager) func(e *core.Reco return err } - trailAuthor, err := e.App.FindFirstRecordByData("activitypub_actors", "id", trail.GetString("author")) - if err != nil { - return err - } - - if err := util.IndexTrail(e.App, trail, trailAuthor, client); err != nil { + if err := util.IndexTrails(e.App, []*core.Record{trail}, client); err != nil { return err } @@ -616,7 +606,7 @@ func createListHandler(client meilisearch.ServiceManager) func(e *core.RecordEve return err } - if err := util.IndexList(e.App, record, author, client); err != nil { + if err := util.IndexLists(e.App, []*core.Record{record}, client); err != nil { return err } @@ -1309,7 +1299,7 @@ func bootstrapCategories(app core.App) error { func bootstrapMeilisearchDocuments(app core.App, client meilisearch.ServiceManager) error { // --- Trails --- - const pageSize = 100 + const pageSize int64 = 100 var page int64 = 0 // Clear index before re-indexing @@ -1321,7 +1311,7 @@ func bootstrapMeilisearchDocuments(app core.App, client meilisearch.ServiceManag trails := []*core.Record{} err := app.RecordQuery("trails"). Limit(pageSize). - Offset(page * int64(pageSize)). + Offset(page * pageSize). All(&trails) if err != nil { return err @@ -1330,70 +1320,9 @@ func bootstrapMeilisearchDocuments(app core.App, client meilisearch.ServiceManag break } - // Collect author IDs to fetch in batch - authorIDs := make([]string, 0, len(trails)) - for _, t := range trails { - if id := t.GetString("author"); id != "" { - authorIDs = append(authorIDs, id) - } - } - - // Fetch all authors in one go - authorArgs := make([]interface{}, len(authorIDs)) - for i, v := range authorIDs { - authorArgs[i] = v - } - - authors, err := app.FindAllRecords("activitypub_actors", - dbx.In("id", authorArgs...), - ) - if err != nil { - return err - } - authorMap := make(map[string]*core.Record, len(authors)) - for _, a := range authors { - authorMap[a.Id] = a - } - - for _, trail := range trails { - author := authorMap[trail.GetString("author")] - - if err := util.IndexTrail(app, trail, author, client); err != nil { - app.Logger().Warn(fmt.Sprintf("Unable to index trail '%s': %v", trail.GetString("name"), err)) - continue - } - - // Shares - shares, err := app.FindAllRecords("trail_share", - dbx.NewExp("trail = {:trailId}", dbx.Params{"trailId": trail.Id}), - ) - if err != nil { - app.Logger().Warn(fmt.Sprintf("Unable to fetch shares for trail '%s': %v", trail.GetString("name"), err)) - continue - } - shareActors := make([]string, len(shares)) - for i, r := range shares { - shareActors[i] = r.GetString("actor") - } - if err := util.UpdateTrailShares(trail.Id, shareActors, client); err != nil { - app.Logger().Warn(fmt.Sprintf("Unable to update trail shares '%s': %v", trail.GetString("name"), err)) - } - - // Likes - likes, err := app.FindAllRecords("trail_like", - dbx.NewExp("trail = {:trailId}", dbx.Params{"trailId": trail.Id}), - ) - if err != nil { - app.Logger().Warn(fmt.Sprintf("Unable to fetch likes for trail '%s': %v", trail.GetString("name"), err)) - continue - } - likeActors := make([]string, len(likes)) - for i, r := range likes { - likeActors[i] = r.GetString("actor") - } - if err := util.UpdateTrailLikes(trail.Id, likeActors, client); err != nil { - app.Logger().Warn(fmt.Sprintf("Unable to update trail likes '%s': %v", trail.GetString("name"), err)) - } + if err := util.IndexTrails(app, trails, client); err != nil { + app.Logger().Warn(fmt.Sprintf("Unable to index trails page %d: %v", page, err)) + continue } page++ @@ -1418,52 +1347,9 @@ func bootstrapMeilisearchDocuments(app core.App, client meilisearch.ServiceManag break } - // Collect author IDs in batch - authorIDs := make([]string, 0, len(lists)) - for _, l := range lists { - if id := l.GetString("author"); id != "" { - authorIDs = append(authorIDs, id) - } - } - // Fetch all authors in one go - authorArgs := make([]interface{}, len(authorIDs)) - for i, v := range authorIDs { - authorArgs[i] = v - } - - authors, err := app.FindAllRecords("activitypub_actors", - dbx.In("id", authorArgs...), - ) - if err != nil { - return err - } - authorMap := make(map[string]*core.Record, len(authors)) - for _, a := range authors { - authorMap[a.Id] = a - } - - for _, list := range lists { - author := authorMap[list.GetString("author")] - - if err := util.IndexList(app, list, author, client); err != nil { - app.Logger().Warn(fmt.Sprintf("Unable to index list '%s': %v", list.GetString("name"), err)) - continue - } - - shares, err := app.FindAllRecords("list_share", - dbx.NewExp("list = {:listId}", dbx.Params{"listId": list.Id}), - ) - if err != nil { - app.Logger().Warn(fmt.Sprintf("Unable to fetch list shares '%s': %v", list.GetString("name"), err)) - continue - } - shareActors := make([]string, len(shares)) - for i, r := range shares { - shareActors[i] = r.GetString("actor") - } - if err := util.UpdateListShares(list.Id, shareActors, client); err != nil { - app.Logger().Warn(fmt.Sprintf("Unable to update list shares '%s': %v", list.GetString("name"), err)) - } + if err := util.IndexLists(app, lists, client); err != nil { + app.Logger().Warn(fmt.Sprintf("Unable to index list page %d: %v", page, err)) + continue } page++ diff --git a/db/util/meilisearch.go b/db/util/meilisearch.go index 98a122a0..a2cb0f0e 100644 --- a/db/util/meilisearch.go +++ b/db/util/meilisearch.go @@ -88,9 +88,33 @@ func documentFromTrailRecord(app core.App, r *core.Record, author *core.Record, } if includeShares { - document["shares"] = []string{} - document["likes"] = []string{} - document["like_count"] = 0 + trailShares := r.ExpandedAll("trail_share_via_trail") + if trailShares != nil { + sharedIDs := make([]string, len(trailShares)) + for i, v := range trailShares { + sharedIDs[i] = v.GetString("actor") + } + + document["shares"] = sharedIDs + + } else { + document["shares"] = []string{} + } + + trailLikes := r.ExpandedAll("trail_like_via_trail") + if trailLikes != nil { + likeIDs := make([]string, len(trailLikes)) + for i, v := range trailLikes { + likeIDs[i] = v.GetString("actor") + } + + document["likes"] = likeIDs + document["like_count"] = len(trailLikes) + + } else { + document["likes"] = []string{} + document["like_count"] = 0 + } } @@ -186,7 +210,7 @@ func documentFromListRecord(r *core.Record, author *core.Record, includeShares b domain = author.GetString("domain") } - document := map[string]interface{}{ + document := map[string]any{ "id": r.Id, "author": author.Id, "author_name": author.GetString("preferred_username"), @@ -206,7 +230,18 @@ func documentFromListRecord(r *core.Record, author *core.Record, includeShares b } if includeShares { - document["shares"] = []string{} + listShares := r.ExpandedAll("list_share_via_list") + if listShares != nil { + sharedIDs := make([]string, len(listShares)) + for i, v := range listShares { + sharedIDs[i] = v.GetString("actor") + } + + document["shares"] = sharedIDs + + } else { + document["shares"] = []string{} + } } return document, nil @@ -266,20 +301,40 @@ func documentFromRemoteRecord(r *core.Record, index string) (map[string]interfac return document, nil } -func IndexTrail(app core.App, r *core.Record, author *core.Record, client meilisearch.ServiceManager) error { - errs := app.ExpandRecord(r, []string{"tags"}, nil) - if len(errs) > 0 { - return fmt.Errorf("failed to expand tags: %v", errs) +func IndexTrails(app core.App, trails []*core.Record, client meilisearch.ServiceManager) error { + documents := make([]map[string]any, len(trails)) + + for i, r := range trails { + errs := app.ExpandRecord(r, []string{"tags"}, nil) + if len(errs) > 0 { + return fmt.Errorf("failed to expand tags: %v", errs) + } + errs = app.ExpandRecord(r, []string{"category"}, nil) + if len(errs) > 0 { + return fmt.Errorf("failed to expand category: %v", errs) + } + errs = app.ExpandRecord(r, []string{"trail_share_via_trail"}, nil) + if len(errs) > 0 { + return fmt.Errorf("failed to expand trail_share_via_trail: %v", errs) + } + errs = app.ExpandRecord(r, []string{"trail_like_via_trail"}, nil) + if len(errs) > 0 { + return fmt.Errorf("failed to expand trail_like_via_trail: %v", errs) + } + errs = app.ExpandRecord(r, []string{"author"}, nil) + if len(errs) > 0 { + return fmt.Errorf("failed to expand author: %v", errs) + } + + author := r.ExpandedOne("author") + + doc, err := documentFromTrailRecord(app, r, author, true) + if err != nil { + return err + } + + documents[i] = doc } - errs = app.ExpandRecord(r, []string{"category"}, nil) - if len(errs) > 0 { - return fmt.Errorf("failed to expand category: %v", errs) - } - doc, err := documentFromTrailRecord(app, r, author, true) - if err != nil { - return err - } - documents := []map[string]interface{}{doc} if _, err := client.Index("trails").AddDocuments(documents); err != nil { return err @@ -338,17 +393,32 @@ func UpdateTrailLikes(trailId string, likes []string, client meilisearch.Service return nil } -func IndexList(app core.App, r *core.Record, author *core.Record, client meilisearch.ServiceManager) error { - errs := app.ExpandRecord(r, []string{"trails"}, nil) - if len(errs) > 0 { - return fmt.Errorf("failed to expand trails: %v", errs) - } +func IndexLists(app core.App, lists []*core.Record, client meilisearch.ServiceManager) error { + documents := make([]map[string]any, len(lists)) - documents, err := documentFromListRecord(r, author, true) - if err != nil { - return err + for i, r := range lists { + errs := app.ExpandRecord(r, []string{"trails"}, nil) + if len(errs) > 0 { + return fmt.Errorf("failed to expand trails: %v", errs) + } + errs = app.ExpandRecord(r, []string{"list_share_via_list"}, nil) + if len(errs) > 0 { + return fmt.Errorf("failed to expand list_share_via_list: %v", errs) + } + errs = app.ExpandRecord(r, []string{"author"}, nil) + if len(errs) > 0 { + return fmt.Errorf("failed to expand author: %v", errs) + } + + author := r.ExpandedOne("author") + + doc, err := documentFromListRecord(r, author, true) + if err != nil { + return err + } + documents[i] = doc } - if _, err = client.Index("lists").AddDocuments(documents); err != nil { + if _, err := client.Index("lists").AddDocuments(documents); err != nil { return err }