improves initial meilisearch indexing speed

db/main.go

@@ -233,7 +233,7 @@ func createTrailHandler(client meilisearch.ServiceManager) func(e *core.RecordEv
 		if err != nil {
 			return err
 		}
-		if err := util.IndexTrail(e.App, record, author, client); err != nil {
+		if err := util.IndexTrails(e.App, []*core.Record{record}, client); err != nil {
 			return err
 		}
 		if !author.GetBool("isLocal") {
@@ -340,12 +340,7 @@ func createSummitLogHandler(client meilisearch.ServiceManager) func(e *core.Reco
 			return err
 		}
 
-		trailAuthor, err := e.App.FindFirstRecordByData("activitypub_actors", "id", trail.GetString("author"))
+		if err := util.IndexTrails(e.App, []*core.Record{trail}, client); err != nil {
-		if err != nil {
-			return err
-		}
-
-		if err := util.IndexTrail(e.App, trail, trailAuthor, client); err != nil {
 			return err
 		}
 
@@ -391,12 +386,7 @@ func deleteSummitLogHandler(client meilisearch.ServiceManager) func(e *core.Reco
 			return err
 		}
 
-		trailAuthor, err := e.App.FindFirstRecordByData("activitypub_actors", "id", trail.GetString("author"))
+		if err := util.IndexTrails(e.App, []*core.Record{trail}, client); err != nil {
-		if err != nil {
-			return err
-		}
-
-		if err := util.IndexTrail(e.App, trail, trailAuthor, client); err != nil {
 			return err
 		}
 
@@ -616,7 +606,7 @@ func createListHandler(client meilisearch.ServiceManager) func(e *core.RecordEve
 			return err
 		}
 
-		if err := util.IndexList(e.App, record, author, client); err != nil {
+		if err := util.IndexLists(e.App, []*core.Record{record}, client); err != nil {
 			return err
 		}
 
@@ -1309,7 +1299,7 @@ func bootstrapCategories(app core.App) error {
 
 func bootstrapMeilisearchDocuments(app core.App, client meilisearch.ServiceManager) error {
 	// --- Trails ---
-	const pageSize = 100
+	const pageSize int64 = 100
 	var page int64 = 0
 
 	// Clear index before re-indexing
@@ -1321,7 +1311,7 @@ func bootstrapMeilisearchDocuments(app core.App, client meilisearch.ServiceManag
 		trails := []*core.Record{}
 		err := app.RecordQuery("trails").
 			Limit(pageSize).
-			Offset(page * int64(pageSize)).
+			Offset(page * pageSize).
 			All(&trails)
 		if err != nil {
 			return err
@@ -1330,72 +1320,11 @@ func bootstrapMeilisearchDocuments(app core.App, client meilisearch.ServiceManag
 			break
 		}
 
-		// Collect author IDs to fetch in batch
+		if err := util.IndexTrails(app, trails, client); err != nil {
-		authorIDs := make([]string, 0, len(trails))
+			app.Logger().Warn(fmt.Sprintf("Unable to index trails page %d: %v", page, err))
-		for _, t := range trails {
-			if id := t.GetString("author"); id != "" {
-				authorIDs = append(authorIDs, id)
-			}
-		}
-
-		// Fetch all authors in one go
-		authorArgs := make([]interface{}, len(authorIDs))
-		for i, v := range authorIDs {
-			authorArgs[i] = v
-		}
-
-		authors, err := app.FindAllRecords("activitypub_actors",
-			dbx.In("id", authorArgs...),
-		)
-		if err != nil {
-			return err
-		}
-		authorMap := make(map[string]*core.Record, len(authors))
-		for _, a := range authors {
-			authorMap[a.Id] = a
-		}
-
-		for _, trail := range trails {
-			author := authorMap[trail.GetString("author")]
-
-			if err := util.IndexTrail(app, trail, author, client); err != nil {
-				app.Logger().Warn(fmt.Sprintf("Unable to index trail '%s': %v", trail.GetString("name"), err))
 			continue
 		}
 
-			// Shares
-			shares, err := app.FindAllRecords("trail_share",
-				dbx.NewExp("trail = {:trailId}", dbx.Params{"trailId": trail.Id}),
-			)
-			if err != nil {
-				app.Logger().Warn(fmt.Sprintf("Unable to fetch shares for trail '%s': %v", trail.GetString("name"), err))
-				continue
-			}
-			shareActors := make([]string, len(shares))
-			for i, r := range shares {
-				shareActors[i] = r.GetString("actor")
-			}
-			if err := util.UpdateTrailShares(trail.Id, shareActors, client); err != nil {
-				app.Logger().Warn(fmt.Sprintf("Unable to update trail shares '%s': %v", trail.GetString("name"), err))
-			}
-
-			// Likes
-			likes, err := app.FindAllRecords("trail_like",
-				dbx.NewExp("trail = {:trailId}", dbx.Params{"trailId": trail.Id}),
-			)
-			if err != nil {
-				app.Logger().Warn(fmt.Sprintf("Unable to fetch likes for trail '%s': %v", trail.GetString("name"), err))
-				continue
-			}
-			likeActors := make([]string, len(likes))
-			for i, r := range likes {
-				likeActors[i] = r.GetString("actor")
-			}
-			if err := util.UpdateTrailLikes(trail.Id, likeActors, client); err != nil {
-				app.Logger().Warn(fmt.Sprintf("Unable to update trail likes '%s': %v", trail.GetString("name"), err))
-			}
-		}
-
 		page++
 	}
 
@@ -1418,54 +1347,11 @@ func bootstrapMeilisearchDocuments(app core.App, client meilisearch.ServiceManag
 			break
 		}
 
-		// Collect author IDs in batch
+		if err := util.IndexLists(app, lists, client); err != nil {
-		authorIDs := make([]string, 0, len(lists))
+			app.Logger().Warn(fmt.Sprintf("Unable to index list page %d: %v", page, err))
-		for _, l := range lists {
-			if id := l.GetString("author"); id != "" {
-				authorIDs = append(authorIDs, id)
-			}
-		}
-		// Fetch all authors in one go
-		authorArgs := make([]interface{}, len(authorIDs))
-		for i, v := range authorIDs {
-			authorArgs[i] = v
-		}
-
-		authors, err := app.FindAllRecords("activitypub_actors",
-			dbx.In("id", authorArgs...),
-		)
-		if err != nil {
-			return err
-		}
-		authorMap := make(map[string]*core.Record, len(authors))
-		for _, a := range authors {
-			authorMap[a.Id] = a
-		}
-
-		for _, list := range lists {
-			author := authorMap[list.GetString("author")]
-
-			if err := util.IndexList(app, list, author, client); err != nil {
-				app.Logger().Warn(fmt.Sprintf("Unable to index list '%s': %v", list.GetString("name"), err))
 			continue
 		}
 
-			shares, err := app.FindAllRecords("list_share",
-				dbx.NewExp("list = {:listId}", dbx.Params{"listId": list.Id}),
-			)
-			if err != nil {
-				app.Logger().Warn(fmt.Sprintf("Unable to fetch list shares '%s': %v", list.GetString("name"), err))
-				continue
-			}
-			shareActors := make([]string, len(shares))
-			for i, r := range shares {
-				shareActors[i] = r.GetString("actor")
-			}
-			if err := util.UpdateListShares(list.Id, shareActors, client); err != nil {
-				app.Logger().Warn(fmt.Sprintf("Unable to update list shares '%s': %v", list.GetString("name"), err))
-			}
-		}
-
 		page++
 	}
 

db/util/meilisearch.go

@@ -88,9 +88,33 @@ func documentFromTrailRecord(app core.App, r *core.Record, author *core.Record,
 	}
 
 	if includeShares {
+		trailShares := r.ExpandedAll("trail_share_via_trail")
+		if trailShares != nil {
+			sharedIDs := make([]string, len(trailShares))
+			for i, v := range trailShares {
+				sharedIDs[i] = v.GetString("actor")
+			}
+
+			document["shares"] = sharedIDs
+
+		} else {
 			document["shares"] = []string{}
+		}
+
+		trailLikes := r.ExpandedAll("trail_like_via_trail")
+		if trailLikes != nil {
+			likeIDs := make([]string, len(trailLikes))
+			for i, v := range trailLikes {
+				likeIDs[i] = v.GetString("actor")
+			}
+
+			document["likes"] = likeIDs
+			document["like_count"] = len(trailLikes)
+
+		} else {
 			document["likes"] = []string{}
 			document["like_count"] = 0
+		}
 
 	}
 
@@ -186,7 +210,7 @@ func documentFromListRecord(r *core.Record, author *core.Record, includeShares b
 		domain = author.GetString("domain")
 	}
 
-	document := map[string]interface{}{
+	document := map[string]any{
 		"id":             r.Id,
 		"author":         author.Id,
 		"author_name":    author.GetString("preferred_username"),
@@ -206,8 +230,19 @@ func documentFromListRecord(r *core.Record, author *core.Record, includeShares b
 	}
 
 	if includeShares {
+		listShares := r.ExpandedAll("list_share_via_list")
+		if listShares != nil {
+			sharedIDs := make([]string, len(listShares))
+			for i, v := range listShares {
+				sharedIDs[i] = v.GetString("actor")
+			}
+
+			document["shares"] = sharedIDs
+
+		} else {
 			document["shares"] = []string{}
 		}
+	}
 
 	return document, nil
 }
@@ -266,7 +301,10 @@ func documentFromRemoteRecord(r *core.Record, index string) (map[string]interfac
 	return document, nil
 }
 
-func IndexTrail(app core.App, r *core.Record, author *core.Record, client meilisearch.ServiceManager) error {
+func IndexTrails(app core.App, trails []*core.Record, client meilisearch.ServiceManager) error {
+	documents := make([]map[string]any, len(trails))
+
+	for i, r := range trails {
 		errs := app.ExpandRecord(r, []string{"tags"}, nil)
 		if len(errs) > 0 {
 			return fmt.Errorf("failed to expand tags: %v", errs)
@@ -275,11 +313,28 @@ func IndexTrail(app core.App, r *core.Record, author *core.Record, client meilis
 		if len(errs) > 0 {
 			return fmt.Errorf("failed to expand category: %v", errs)
 		}
+		errs = app.ExpandRecord(r, []string{"trail_share_via_trail"}, nil)
+		if len(errs) > 0 {
+			return fmt.Errorf("failed to expand trail_share_via_trail: %v", errs)
+		}
+		errs = app.ExpandRecord(r, []string{"trail_like_via_trail"}, nil)
+		if len(errs) > 0 {
+			return fmt.Errorf("failed to expand trail_like_via_trail: %v", errs)
+		}
+		errs = app.ExpandRecord(r, []string{"author"}, nil)
+		if len(errs) > 0 {
+			return fmt.Errorf("failed to expand author: %v", errs)
+		}
+
+		author := r.ExpandedOne("author")
+
 		doc, err := documentFromTrailRecord(app, r, author, true)
 		if err != nil {
 			return err
 		}
-	documents := []map[string]interface{}{doc}
+
+		documents[i] = doc
+	}
 
 	if _, err := client.Index("trails").AddDocuments(documents); err != nil {
 		return err
@@ -338,17 +393,32 @@ func UpdateTrailLikes(trailId string, likes []string, client meilisearch.Service
 	return nil
 }
 
-func IndexList(app core.App, r *core.Record, author *core.Record, client meilisearch.ServiceManager) error {
+func IndexLists(app core.App, lists []*core.Record, client meilisearch.ServiceManager) error {
+	documents := make([]map[string]any, len(lists))
+
+	for i, r := range lists {
 		errs := app.ExpandRecord(r, []string{"trails"}, nil)
 		if len(errs) > 0 {
 			return fmt.Errorf("failed to expand trails: %v", errs)
 		}
+		errs = app.ExpandRecord(r, []string{"list_share_via_list"}, nil)
+		if len(errs) > 0 {
+			return fmt.Errorf("failed to expand list_share_via_list: %v", errs)
+		}
+		errs = app.ExpandRecord(r, []string{"author"}, nil)
+		if len(errs) > 0 {
+			return fmt.Errorf("failed to expand author: %v", errs)
+		}
 
-	documents, err := documentFromListRecord(r, author, true)
+		author := r.ExpandedOne("author")
+
+		doc, err := documentFromListRecord(r, author, true)
 		if err != nil {
 			return err
 		}
-	if _, err = client.Index("lists").AddDocuments(documents); err != nil {
+		documents[i] = doc
+	}
+	if _, err := client.Index("lists").AddDocuments(documents); err != nil {
 		return err
 	}