feat: implement server-side map clustering (#991)
* feat: meilisearch bounding box intersection and map filtering * feat: tiered clustering and polyline filtering based on bounding box diagonal Addressing review comments with optimized performance and data accuracy: - Implemented a two-tiered search strategy (Summary vs. Detailed) to provide 100% accurate cluster counts while minimizing metadata traffic. - Added a client-side ID-based cache to eliminate redundant network requests for trails already in memory. - Introduced tiered 'detail shedding' that dynamically hides small trail lines when zooming out to maintain smooth panning performance. - Fixed Svelte 5 reactivity issues to ensure clusters reappear instantly when trail detail is shed. - Consolidated zoom thresholds into centralized constants for system-wide consistency. - Updated Meilisearch configuration to support efficient ID-based filtering. * feat: implement server-side map clustering using Meilisearch and Supercluster - Implemented SvelteKit server-side clustering route at `/api/v1/search/trails/cluster` using the `supercluster` library. - Integrated backend search results with the new clustering endpoint to improve performance for large trail datasets. - Fixed Svelte 5 reactivity in `MapWithElevationMaplibre.svelte` by making `mapLoaded` reactive and fixing destructuring of `` map data. - Removed obsolete client-side zoom constraints (`minzoom`/`maxzoom`) from `TrailLayer`, `PreviewLayer`, and `ClusterLayer` to allow dynamic visibility controlled by backend attributes. - Fixed a bug where unauthenticated users generated invalid Meilisearch filters. - Optimized map page performance by caching bounding box and filter values in the route loader. - Updated `ClusterLayer` font to `Noto Sans Regular` to match available tileserver resources. * feat: implement dynamic polyline visibility based on result density - Replaced static zoom-based diagonal thresholds with a dynamic "Top N" approach for polyline visibility. - Updated `/api/v1/search/trails/cluster` to mark only the top `MAP_MAX_POLYLINES` (default 100) trails by bounding box diagonal as "large". - Modified `trails_search_bounding_box` store function to always fetch polylines for trails marked as large, regardless of zoom level. - Removed obsolete `MAP_*_ZOOM_DIAGONAL_LIMIT` constants and simplified map layer constructors by removing tier-based filtering. - Updated environment configurations and documentation to use the new `PUBLIC_MAP_MAX_POLYLINES` variable. * refactor: remove obsolete zoom-based map clustering thresholds Following the transition to dynamic server-side clustering based on result density (Top N polylines), this commit removes all remaining zoom-based thresholds and logic. - Removed PUBLIC_MAP_LOW_ZOOM_THRESHOLD, PUBLIC_MAP_MEDIUM_ZOOM_THRESHOLD, and PUBLIC_MAP_HIGH_ZOOM_THRESHOLD environment variables. - Removed the 'mapClusterMinZoom' user setting from the schema, models, and settings UI. - Simplified the map component and MapLibre layer managers by removing unused 'clusterMinZoom', 'minZoom', and 'maxZoom' parameters. - Cleaned up obsolete 'map-cluster-zoom-level' translation keys across all locales. - Updated documentation to reflect the removal of these variables. * chore: remove map clustering debug logs * docs: update changelog and fix global zoom feature disappearance * fix: address PR reviews and refine map cluster/preview visuals * feat: implement configurable map cluster zoom and trail start marker settings * feat: render single unclustered shedded trails as start markers instead of cluster circles * minor fixes * minor ui changes * fix missing entries in map list * remove redundant map clustering zoom clamp * fix merge issues * fix merge issues * fixes * hide popup for trails without details * several improvements * prevent import of client trail_store in server cluster code * update changelog --------- Co-authored-by: slothful-vassal <89943360+slothful-vassal@users.noreply.github.com>
This commit is contained in:
Pål Håland
218
db/migrations/1778583800_persist_trail_bounds.go
Normal file
218
db/migrations/1778583800_persist_trail_bounds.go
Normal file
@@ -0,0 +1,218 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"pocketbase/util"
|
||||
|
||||
"github.com/pocketbase/dbx"
|
||||
"github.com/pocketbase/pocketbase/core"
|
||||
m "github.com/pocketbase/pocketbase/migrations"
|
||||
)
|
||||
|
||||
const trailBoundsViewQuery = `SELECT
|
||||
a.id, a.user,
|
||||
COALESCE(MAX(t.max_lat), 0) AS max_lat,
|
||||
COALESCE(MAX(t.max_lon), 0) AS max_lon,
|
||||
COALESCE(MIN(t.min_lat), 0) AS min_lat,
|
||||
COALESCE(MIN(t.min_lon), 0) AS min_lon
|
||||
FROM activitypub_actors a
|
||||
LEFT JOIN (
|
||||
SELECT author AS actor_id,
|
||||
MAX(max_lat) AS max_lat,
|
||||
MAX(max_lon) AS max_lon,
|
||||
MIN(min_lat) AS min_lat,
|
||||
MIN(min_lon) AS min_lon
|
||||
FROM trails
|
||||
GROUP BY author
|
||||
UNION ALL
|
||||
SELECT ts.actor AS actor_id,
|
||||
MAX(t.max_lat) AS max_lat,
|
||||
MAX(t.max_lon) AS max_lon,
|
||||
MIN(t.min_lat) AS min_lat,
|
||||
MIN(t.min_lon) AS min_lon
|
||||
FROM trail_share ts
|
||||
JOIN trails t ON t.id = ts.trail
|
||||
GROUP BY ts.actor
|
||||
UNION ALL
|
||||
SELECT a2.id AS actor_id,
|
||||
p.max_lat, p.max_lon, p.min_lat, p.min_lon
|
||||
FROM activitypub_actors a2
|
||||
CROSS JOIN (
|
||||
SELECT
|
||||
MAX(max_lat) AS max_lat,
|
||||
MAX(max_lon) AS max_lon,
|
||||
MIN(min_lat) AS min_lat,
|
||||
MIN(min_lon) AS min_lon
|
||||
FROM trails
|
||||
WHERE public = TRUE
|
||||
) p
|
||||
) t ON t.actor_id = a.id
|
||||
WHERE a.user != ""
|
||||
GROUP BY a.id;`
|
||||
|
||||
const trailStartPointBoundsViewQuery = `SELECT
|
||||
a.id, a.user,
|
||||
COALESCE(MAX(t.max_lat), 0) AS max_lat,
|
||||
COALESCE(MAX(t.max_lon), 0) AS max_lon,
|
||||
COALESCE(MIN(t.min_lat), 0) AS min_lat,
|
||||
COALESCE(MIN(t.min_lon), 0) AS min_lon
|
||||
FROM activitypub_actors a
|
||||
LEFT JOIN (
|
||||
SELECT author AS actor_id,
|
||||
MAX(lat) AS max_lat,
|
||||
MAX(lon) AS max_lon,
|
||||
MIN(lat) AS min_lat,
|
||||
MIN(lon) AS min_lon
|
||||
FROM trails
|
||||
GROUP BY author
|
||||
UNION ALL
|
||||
SELECT ts.actor AS actor_id,
|
||||
MAX(t.lat) AS max_lat,
|
||||
MAX(t.lon) AS max_lon,
|
||||
MIN(t.lat) AS min_lat,
|
||||
MIN(t.lon) AS min_lon
|
||||
FROM trail_share ts
|
||||
JOIN trails t ON t.id = ts.trail
|
||||
GROUP BY ts.actor
|
||||
UNION ALL
|
||||
SELECT a2.id AS actor_id,
|
||||
p.max_lat, p.max_lon, p.min_lat, p.min_lon
|
||||
FROM activitypub_actors a2
|
||||
CROSS JOIN (
|
||||
SELECT
|
||||
MAX(lat) AS max_lat,
|
||||
MAX(lon) AS max_lon,
|
||||
MIN(lat) AS min_lat,
|
||||
MIN(lon) AS min_lon
|
||||
FROM trails
|
||||
WHERE public = TRUE
|
||||
) p
|
||||
) t ON t.actor_id = a.id
|
||||
WHERE a.user != ""
|
||||
GROUP BY a.id;`
|
||||
|
||||
func init() {
|
||||
m.Register(func(app core.App) error {
|
||||
trailsCollection, err := app.FindCollectionByNameOrId("trails")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
addTrailBoundsField(trailsCollection, "min_lat")
|
||||
addTrailBoundsField(trailsCollection, "max_lat")
|
||||
addTrailBoundsField(trailsCollection, "min_lon")
|
||||
addTrailBoundsField(trailsCollection, "max_lon")
|
||||
addTrailBoundsField(trailsCollection, "bounding_box_diagonal")
|
||||
|
||||
if err := app.Save(trailsCollection); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := backfillTrailBounds(app); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
boundingBoxCollection, err := app.FindCollectionByNameOrId("trails_bounding_box")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := json.Unmarshal([]byte(`{"viewQuery":`+strconvQuote(trailBoundsViewQuery)+`}`), &boundingBoxCollection); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := app.Save(boundingBoxCollection); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}, func(app core.App) error {
|
||||
boundingBoxCollection, err := app.FindCollectionByNameOrId("trails_bounding_box")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := json.Unmarshal([]byte(`{"viewQuery":`+strconvQuote(trailStartPointBoundsViewQuery)+`}`), &boundingBoxCollection); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := app.Save(boundingBoxCollection); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
trailsCollection, err := app.FindCollectionByNameOrId("trails")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
trailsCollection.Fields.RemoveByName("min_lat")
|
||||
trailsCollection.Fields.RemoveByName("max_lat")
|
||||
trailsCollection.Fields.RemoveByName("min_lon")
|
||||
trailsCollection.Fields.RemoveByName("max_lon")
|
||||
trailsCollection.Fields.RemoveByName("bounding_box_diagonal")
|
||||
|
||||
if err := app.Save(trailsCollection); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func addTrailBoundsField(collection *core.Collection, name string) {
|
||||
if collection.Fields.GetByName(name) != nil {
|
||||
return
|
||||
}
|
||||
|
||||
collection.Fields.Add(&core.NumberField{
|
||||
Name: name,
|
||||
})
|
||||
}
|
||||
|
||||
func backfillTrailBounds(app core.App) error {
|
||||
const pageSize int64 = 50
|
||||
lastID := ""
|
||||
|
||||
for {
|
||||
trails := []*core.Record{}
|
||||
query := app.RecordQuery("trails").
|
||||
OrderBy("id ASC").
|
||||
Limit(pageSize)
|
||||
|
||||
if lastID != "" {
|
||||
query = query.AndWhere(dbx.NewExp("id > {:lastID}", dbx.Params{"lastID": lastID}))
|
||||
}
|
||||
|
||||
if err := query.All(&trails); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(trails) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, trail := range trails {
|
||||
if err := util.SavePolyline(app, trail); err != nil {
|
||||
if err := saveDefaultTrailBounds(app, trail); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
lastID = trail.Id
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func saveDefaultTrailBounds(app core.App, trail *core.Record) error {
|
||||
lat := trail.GetFloat("lat")
|
||||
lon := trail.GetFloat("lon")
|
||||
trail.Set("min_lat", lat)
|
||||
trail.Set("max_lat", lat)
|
||||
trail.Set("min_lon", lon)
|
||||
trail.Set("max_lon", lon)
|
||||
trail.Set("bounding_box_diagonal", 0)
|
||||
return app.UnsafeWithoutHooks().Save(trail)
|
||||
}
|
||||
|
||||
func strconvQuote(value string) string {
|
||||
raw, _ := json.Marshal(value)
|
||||
return string(raw)
|
||||
}
|
||||
Reference in New Issue
Block a user