feat: implement server-side map clustering (#991)

* feat: meilisearch bounding box intersection and map filtering

* feat: tiered clustering and polyline filtering based on bounding box diagonal

Addressing review comments with optimized performance and data accuracy:
- Implemented a two-tiered search strategy (Summary vs. Detailed) to provide 100% accurate cluster counts while minimizing metadata traffic.
- Added a client-side ID-based cache to eliminate redundant network requests for trails already in memory.
- Introduced tiered 'detail shedding' that dynamically hides small trail lines when zooming out to maintain smooth panning performance.
- Fixed Svelte 5 reactivity issues to ensure clusters reappear instantly when trail detail is shed.
- Consolidated zoom thresholds into centralized constants for system-wide consistency.
- Updated Meilisearch configuration to support efficient ID-based filtering.

* feat: implement server-side map clustering using Meilisearch and Supercluster

- Implemented SvelteKit server-side clustering route at `/api/v1/search/trails/cluster` using the `supercluster` library.
- Integrated backend search results with the new clustering endpoint to improve performance for large trail datasets.
- Fixed Svelte 5 reactivity in `MapWithElevationMaplibre.svelte` by making `mapLoaded` reactive and fixing destructuring of `` map data.
- Removed obsolete client-side zoom constraints (`minzoom`/`maxzoom`) from `TrailLayer`, `PreviewLayer`, and `ClusterLayer` to allow dynamic visibility controlled by backend attributes.
- Fixed a bug where unauthenticated users generated invalid Meilisearch filters.
- Optimized map page performance by caching bounding box and filter values in the route loader.
- Updated `ClusterLayer` font to `Noto Sans Regular` to match available tileserver resources.

* feat: implement dynamic polyline visibility based on result density

- Replaced static zoom-based diagonal thresholds with a dynamic "Top N" approach for polyline visibility.
- Updated `/api/v1/search/trails/cluster` to mark only the top `MAP_MAX_POLYLINES` (default 100) trails by bounding box diagonal as "large".
- Modified `trails_search_bounding_box` store function to always fetch polylines for trails marked as large, regardless of zoom level.
- Removed obsolete `MAP_*_ZOOM_DIAGONAL_LIMIT` constants and simplified map layer constructors by removing tier-based filtering.
- Updated environment configurations and documentation to use the new `PUBLIC_MAP_MAX_POLYLINES` variable.

* refactor: remove obsolete zoom-based map clustering thresholds

Following the transition to dynamic server-side clustering based on result density (Top N polylines), this commit removes all remaining zoom-based thresholds and logic.

- Removed PUBLIC_MAP_LOW_ZOOM_THRESHOLD, PUBLIC_MAP_MEDIUM_ZOOM_THRESHOLD, and PUBLIC_MAP_HIGH_ZOOM_THRESHOLD environment variables.
- Removed the 'mapClusterMinZoom' user setting from the schema, models, and settings UI.
- Simplified the map component and MapLibre layer managers by removing unused 'clusterMinZoom', 'minZoom', and 'maxZoom' parameters.
- Cleaned up obsolete 'map-cluster-zoom-level' translation keys across all locales.
- Updated documentation to reflect the removal of these variables.

* chore: remove map clustering debug logs

* docs: update changelog and fix global zoom feature disappearance

* fix: address PR reviews and refine map cluster/preview visuals

* feat: implement configurable map cluster zoom and trail start marker settings

* feat: render single unclustered shedded trails as start markers instead of cluster circles

* minor fixes

* minor ui changes

* fix missing entries in map list

* remove redundant map clustering zoom clamp

* fix merge issues

* fix merge issues

* fixes

* hide popup for trails without details

* several improvements

* prevent import of client trail_store in server cluster code

* update changelog

---------

Co-authored-by: slothful-vassal <89943360+slothful-vassal@users.noreply.github.com>

db/main.go

@@ -299,9 +299,9 @@ func initMeilisearchConfig(client meilisearch.ServiceManager) {
 		"trails": {
 			SearchableAttributes: []string{"author_name", "name", "description", "location", "tags"},
 			FilterableAttributes: []string{
-				"_geo", "author", "category", "completed", "date", "difficulty",
+				"id", "_geo", "author", "category", "completed", "date", "difficulty",
 				"distance", "elevation_gain", "elevation_loss", "likes", "public",
-				"shares", "tags",
+				"shares", "tags", "min_lat", "max_lat", "min_lon", "max_lon", "bounding_box_diagonal",
 			},
 			SortableAttributes: []string{
 				"author", "created", "date", "difficulty", "distance",

db/migrations/1778583800_persist_trail_bounds.go

@@ -0,0 +1,218 @@
+package migrations
+
+import (
+	"encoding/json"
+	"pocketbase/util"
+
+	"github.com/pocketbase/dbx"
+	"github.com/pocketbase/pocketbase/core"
+	m "github.com/pocketbase/pocketbase/migrations"
+)
+
+const trailBoundsViewQuery = `SELECT 
+    a.id, a.user, 
+    COALESCE(MAX(t.max_lat), 0) AS max_lat, 
+    COALESCE(MAX(t.max_lon), 0) AS max_lon, 
+    COALESCE(MIN(t.min_lat), 0) AS min_lat, 
+    COALESCE(MIN(t.min_lon), 0) AS min_lon 
+FROM activitypub_actors a 
+LEFT JOIN ( 
+    SELECT author AS actor_id, 
+        MAX(max_lat) AS max_lat, 
+        MAX(max_lon) AS max_lon, 
+        MIN(min_lat) AS min_lat, 
+        MIN(min_lon) AS min_lon 
+    FROM trails 
+    GROUP BY author 
+    UNION ALL 
+    SELECT ts.actor AS actor_id, 
+        MAX(t.max_lat) AS max_lat, 
+        MAX(t.max_lon) AS max_lon, 
+        MIN(t.min_lat) AS min_lat, 
+        MIN(t.min_lon) AS min_lon 
+    FROM trail_share ts 
+    JOIN trails t ON t.id = ts.trail 
+    GROUP BY ts.actor 
+    UNION ALL 
+    SELECT a2.id AS actor_id, 
+        p.max_lat, p.max_lon, p.min_lat, p.min_lon 
+    FROM activitypub_actors a2 
+    CROSS JOIN ( 
+        SELECT 
+            MAX(max_lat) AS max_lat, 
+            MAX(max_lon) AS max_lon, 
+            MIN(min_lat) AS min_lat, 
+            MIN(min_lon) AS min_lon 
+        FROM trails 
+        WHERE public = TRUE 
+    ) p 
+) t ON t.actor_id = a.id 
+WHERE a.user != ""
+GROUP BY a.id;`
+
+const trailStartPointBoundsViewQuery = `SELECT 
+    a.id, a.user, 
+    COALESCE(MAX(t.max_lat), 0) AS max_lat, 
+    COALESCE(MAX(t.max_lon), 0) AS max_lon, 
+    COALESCE(MIN(t.min_lat), 0) AS min_lat, 
+    COALESCE(MIN(t.min_lon), 0) AS min_lon 
+FROM activitypub_actors a 
+LEFT JOIN ( 
+    SELECT author AS actor_id, 
+        MAX(lat) AS max_lat, 
+        MAX(lon) AS max_lon, 
+        MIN(lat) AS min_lat, 
+        MIN(lon) AS min_lon 
+    FROM trails 
+    GROUP BY author 
+    UNION ALL 
+    SELECT ts.actor AS actor_id, 
+        MAX(t.lat) AS max_lat, 
+        MAX(t.lon) AS max_lon, 
+        MIN(t.lat) AS min_lat, 
+        MIN(t.lon) AS min_lon 
+    FROM trail_share ts 
+    JOIN trails t ON t.id = ts.trail 
+    GROUP BY ts.actor 
+    UNION ALL 
+    SELECT a2.id AS actor_id, 
+        p.max_lat, p.max_lon, p.min_lat, p.min_lon 
+    FROM activitypub_actors a2 
+    CROSS JOIN ( 
+        SELECT 
+            MAX(lat) AS max_lat, 
+            MAX(lon) AS max_lon, 
+            MIN(lat) AS min_lat, 
+            MIN(lon) AS min_lon 
+        FROM trails 
+        WHERE public = TRUE 
+    ) p 
+) t ON t.actor_id = a.id 
+WHERE a.user != ""
+GROUP BY a.id;`
+
+func init() {
+	m.Register(func(app core.App) error {
+		trailsCollection, err := app.FindCollectionByNameOrId("trails")
+		if err != nil {
+			return err
+		}
+
+		addTrailBoundsField(trailsCollection, "min_lat")
+		addTrailBoundsField(trailsCollection, "max_lat")
+		addTrailBoundsField(trailsCollection, "min_lon")
+		addTrailBoundsField(trailsCollection, "max_lon")
+		addTrailBoundsField(trailsCollection, "bounding_box_diagonal")
+
+		if err := app.Save(trailsCollection); err != nil {
+			return err
+		}
+
+		if err := backfillTrailBounds(app); err != nil {
+			return err
+		}
+
+		boundingBoxCollection, err := app.FindCollectionByNameOrId("trails_bounding_box")
+		if err != nil {
+			return err
+		}
+
+		if err := json.Unmarshal([]byte(`{"viewQuery":`+strconvQuote(trailBoundsViewQuery)+`}`), &boundingBoxCollection); err != nil {
+			return err
+		}
+
+		if err := app.Save(boundingBoxCollection); err != nil {
+			return err
+		}
+
+		return nil
+	}, func(app core.App) error {
+		boundingBoxCollection, err := app.FindCollectionByNameOrId("trails_bounding_box")
+		if err != nil {
+			return err
+		}
+
+		if err := json.Unmarshal([]byte(`{"viewQuery":`+strconvQuote(trailStartPointBoundsViewQuery)+`}`), &boundingBoxCollection); err != nil {
+			return err
+		}
+
+		if err := app.Save(boundingBoxCollection); err != nil {
+			return err
+		}
+
+		trailsCollection, err := app.FindCollectionByNameOrId("trails")
+		if err != nil {
+			return err
+		}
+
+		trailsCollection.Fields.RemoveByName("min_lat")
+		trailsCollection.Fields.RemoveByName("max_lat")
+		trailsCollection.Fields.RemoveByName("min_lon")
+		trailsCollection.Fields.RemoveByName("max_lon")
+		trailsCollection.Fields.RemoveByName("bounding_box_diagonal")
+
+		if err := app.Save(trailsCollection); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func addTrailBoundsField(collection *core.Collection, name string) {
+	if collection.Fields.GetByName(name) != nil {
+		return
+	}
+
+	collection.Fields.Add(&core.NumberField{
+		Name: name,
+	})
+}
+
+func backfillTrailBounds(app core.App) error {
+	const pageSize int64 = 50
+	lastID := ""
+
+	for {
+		trails := []*core.Record{}
+		query := app.RecordQuery("trails").
+			OrderBy("id ASC").
+			Limit(pageSize)
+
+		if lastID != "" {
+			query = query.AndWhere(dbx.NewExp("id > {:lastID}", dbx.Params{"lastID": lastID}))
+		}
+
+		if err := query.All(&trails); err != nil {
+			return err
+		}
+		if len(trails) == 0 {
+			return nil
+		}
+
+		for _, trail := range trails {
+			if err := util.SavePolyline(app, trail); err != nil {
+				if err := saveDefaultTrailBounds(app, trail); err != nil {
+					return err
+				}
+			}
+			lastID = trail.Id
+		}
+	}
+}
+
+func saveDefaultTrailBounds(app core.App, trail *core.Record) error {
+	lat := trail.GetFloat("lat")
+	lon := trail.GetFloat("lon")
+	trail.Set("min_lat", lat)
+	trail.Set("max_lat", lat)
+	trail.Set("min_lon", lon)
+	trail.Set("max_lon", lon)
+	trail.Set("bounding_box_diagonal", 0)
+	return app.UnsafeWithoutHooks().Save(trail)
+}
+
+func strconvQuote(value string) string {
+	raw, _ := json.Marshal(value)
+	return string(raw)
+}

db/util/meilisearch.go

@@ -39,35 +39,47 @@ func documentFromTrailRecord(app core.App, r *core.Record, author *core.Record,
 		category = trailCategory.GetString("name")
 	}
 
+	bounds := getStoredBounds(r)
+
 	domain := ""
 	if !author.GetBool("isLocal") {
 		domain = author.GetString("domain")
 	}
 
+	diagonal := r.GetFloat("bounding_box_diagonal")
+	if diagonal == 0 && (bounds[0] != bounds[1] || bounds[2] != bounds[3]) {
+		diagonal = HaversineDistance(bounds[0], bounds[2], bounds[1], bounds[3])
+	}
+
 	document := map[string]any{
-		"id":             r.Id,
-		"author":         author.Id,
-		"author_name":    author.GetString("preferred_username"),
-		"author_avatar":  author.GetString("icon"),
-		"name":           r.GetString("name"),
-		"description":    r.GetString("description"),
-		"location":       r.GetString("location"),
-		"distance":       r.GetFloat("distance"),
-		"elevation_gain": r.GetFloat("elevation_gain"),
-		"elevation_loss": r.GetFloat("elevation_loss"),
-		"duration":       r.GetFloat("duration"),
-		"difficulty":     difficultyToNumber(r.GetString("difficulty")),
-		"category":       category,
-		"completed":      r.GetBool("completed"),
-		"date":           r.GetDateTime("date").Time().Unix(),
-		"created":        r.GetDateTime("created").Time().Unix(),
-		"public":         r.GetBool("public"),
-		"thumbnail":      thumbnail,
-		"gpx":            r.GetString("gpx"),
-		"tags":           tags,
-		"polyline":       r.GetString("polyline"),
-		"domain":         domain,
-		"iri":            r.GetString("iri"),
+		"id":                    r.Id,
+		"author":                author.Id,
+		"author_name":           author.GetString("preferred_username"),
+		"author_avatar":         author.GetString("icon"),
+		"name":                  r.GetString("name"),
+		"description":           r.GetString("description"),
+		"location":              r.GetString("location"),
+		"distance":              r.GetFloat("distance"),
+		"elevation_gain":        r.GetFloat("elevation_gain"),
+		"elevation_loss":        r.GetFloat("elevation_loss"),
+		"duration":              r.GetFloat("duration"),
+		"difficulty":            difficultyToNumber(r.GetString("difficulty")),
+		"category":              category,
+		"completed":             r.GetBool("completed"),
+		"date":                  r.GetDateTime("date").Time().Unix(),
+		"created":               r.GetDateTime("created").Time().Unix(),
+		"public":                r.GetBool("public"),
+		"thumbnail":             thumbnail,
+		"gpx":                   r.GetString("gpx"),
+		"tags":                  tags,
+		"polyline":              r.GetString("polyline"),
+		"domain":                domain,
+		"iri":                   r.GetString("iri"),
+		"min_lat":               bounds[0],
+		"max_lat":               bounds[1],
+		"min_lon":               bounds[2],
+		"max_lon":               bounds[3],
+		"bounding_box_diagonal": diagonal,
 		"_geo": map[string]float64{
 			"lat": r.GetFloat("lat"),
 			"lng": r.GetFloat("lon"),
@@ -121,6 +133,22 @@ func difficultyToNumber(difficulty string) int32 {
 	return 0
 }
 
+func getStoredBounds(r *core.Record) [4]float64 {
+	lat := r.GetFloat("lat")
+	lon := r.GetFloat("lon")
+	defaultBounds := [4]float64{lat, lat, lon, lon}
+
+	minLat := r.GetFloat("min_lat")
+	maxLat := r.GetFloat("max_lat")
+	minLon := r.GetFloat("min_lon")
+	maxLon := r.GetFloat("max_lon")
+	if minLat == 0 && maxLat == 0 && minLon == 0 && maxLon == 0 && (lat != 0 || lon != 0) {
+		return defaultBounds
+	}
+
+	return [4]float64{minLat, maxLat, minLon, maxLon}
+}
+
 func documentFromListRecord(r *core.Record, author *core.Record, includeShares bool) (map[string]any, error) {
 
 	totalElevationGain := 0.0

db/util/polyline.go

@@ -12,33 +12,82 @@ import (
 
 const PolylineMaxLength = 5 * 1024 * 1024
 
-func ComputePolyline(app core.App, r *core.Record) (string, error) {
+type TrailGeometry struct {
+	Polyline            string
+	MinLat              float64
+	MaxLat              float64
+	MinLon              float64
+	MaxLon              float64
+	BoundingBoxDiagonal float64
+}
+
+func ComputeTrailGeometry(app core.App, r *core.Record) (*TrailGeometry, error) {
+	geometry := &TrailGeometry{
+		MinLat: r.GetFloat("lat"),
+		MaxLat: r.GetFloat("lat"),
+		MinLon: r.GetFloat("lon"),
+		MaxLon: r.GetFloat("lon"),
+	}
+
 	gpxPath := r.GetString("gpx")
 	if len(gpxPath) == 0 {
-		return "", nil
+		return geometry, nil
 	}
 
 	fsys, err := app.NewFilesystem()
 	if err != nil {
-		return "", fmt.Errorf("open filesystem: %w", err)
+		return nil, fmt.Errorf("open filesystem: %w", err)
 	}
 	defer fsys.Close()
 
 	gpxFilePath := r.BaseFilesPath() + "/" + gpxPath
 	gpxFile, err := fsys.GetReader(gpxFilePath)
 	if err != nil {
-		return "", fmt.Errorf("open gpx file %q: %w", gpxFilePath, err)
+		return nil, fmt.Errorf("open gpx file %q: %w", gpxFilePath, err)
 	}
 	defer gpxFile.Close()
 
 	content := new(bytes.Buffer)
 	if _, err = io.Copy(content, gpxFile); err != nil {
-		return "", fmt.Errorf("read gpx file %q: %w", gpxFilePath, err)
+		return nil, fmt.Errorf("read gpx file %q: %w", gpxFilePath, err)
 	}
 
 	gpxData, err := gpx.Parse(content)
 	if err != nil {
-		return "", fmt.Errorf("parse gpx file %q: %w", gpxFilePath, err)
+		return nil, fmt.Errorf("parse gpx file %q: %w", gpxFilePath, err)
+	}
+
+	minLat, maxLat, minLon, maxLon := 90.0, -90.0, 180.0, -180.0
+	hasPoints := false
+
+	addPoint := func(lat, lon float64) {
+		if lat < minLat {
+			minLat = lat
+		}
+		if lat > maxLat {
+			maxLat = lat
+		}
+		if lon < minLon {
+			minLon = lon
+		}
+		if lon > maxLon {
+			maxLon = lon
+		}
+		hasPoints = true
+	}
+
+	for _, trk := range gpxData.Tracks {
+		for _, seg := range trk.Segments {
+			for _, pt := range seg.Points {
+				addPoint(pt.Latitude, pt.Longitude)
+			}
+		}
+	}
+
+	for _, rte := range gpxData.Routes {
+		for _, pt := range rte.Points {
+			addPoint(pt.Latitude, pt.Longitude)
+		}
 	}
 
 	gpxData.SimplifyTracks(50)
@@ -50,21 +99,44 @@ func ComputePolyline(app core.App, r *core.Record) (string, error) {
 			}
 		}
 	}
-	return string(polyline.EncodeCoords(coordinates)), nil
+	geometry.Polyline = string(polyline.EncodeCoords(coordinates))
+
+	if hasPoints {
+		geometry.MinLat = minLat
+		geometry.MaxLat = maxLat
+		geometry.MinLon = minLon
+		geometry.MaxLon = maxLon
+		geometry.BoundingBoxDiagonal = HaversineDistance(minLat, minLon, maxLat, maxLon)
+	}
+
+	return geometry, nil
+}
+
+func ComputePolyline(app core.App, r *core.Record) (string, error) {
+	geometry, err := ComputeTrailGeometry(app, r)
+	if err != nil {
+		return "", err
+	}
+	return geometry.Polyline, nil
 }
 
 func SavePolyline(app core.App, r *core.Record) error {
-	encoded, err := ComputePolyline(app, r)
+	geometry, err := ComputeTrailGeometry(app, r)
 	if err != nil {
 		return err
 	}
 	// Encoded polylines are ASCII-only, so byte length matches character length.
-	if len(encoded) > PolylineMaxLength {
+	if len(geometry.Polyline) > PolylineMaxLength {
 		return fmt.Errorf("polyline exceeds maximum length of %d characters", PolylineMaxLength)
 	}
-	r.Set("polyline", encoded)
+	r.Set("polyline", geometry.Polyline)
+	r.Set("min_lat", geometry.MinLat)
+	r.Set("max_lat", geometry.MaxLat)
+	r.Set("min_lon", geometry.MinLon)
+	r.Set("max_lon", geometry.MaxLon)
+	r.Set("bounding_box_diagonal", geometry.BoundingBoxDiagonal)
 	if err := app.UnsafeWithoutHooks().Save(r); err != nil {
-		return fmt.Errorf("save trail polyline: %w", err)
+		return fmt.Errorf("save trail geometry: %w", err)
 	}
 	return nil
 }