feat: implement server-side map clustering (#991)

* feat: meilisearch bounding box intersection and map filtering * feat: tiered clustering and polyline filtering based on bounding box diagonal Addressing review comments with optimized performance and data accuracy: - Implemented a two-tiered search strategy (Summary vs. Detailed) to provide 100% accurate cluster counts while minimizing metadata traffic. - Added a client-side ID-based cache to eliminate redundant network requests for trails already in memory. - Introduced tiered 'detail shedding' that dynamically hides small trail lines when zooming out to maintain smooth panning performance. - Fixed Svelte 5 reactivity issues to ensure clusters reappear instantly when trail detail is shed. - Consolidated zoom thresholds into centralized constants for system-wide consistency. - Updated Meilisearch configuration to support efficient ID-based filtering. * feat: implement server-side map clustering using Meilisearch and Supercluster - Implemented SvelteKit server-side clustering route at `/api/v1/search/trails/cluster` using the `supercluster` library. - Integrated backend search results with the new clustering endpoint to improve performance for large trail datasets. - Fixed Svelte 5 reactivity in `MapWithElevationMaplibre.svelte` by making `mapLoaded` reactive and fixing destructuring of `` map data. - Removed obsolete client-side zoom constraints (`minzoom`/`maxzoom`) from `TrailLayer`, `PreviewLayer`, and `ClusterLayer` to allow dynamic visibility controlled by backend attributes. - Fixed a bug where unauthenticated users generated invalid Meilisearch filters. - Optimized map page performance by caching bounding box and filter values in the route loader. - Updated `ClusterLayer` font to `Noto Sans Regular` to match available tileserver resources. * feat: implement dynamic polyline visibility based on result density - Replaced static zoom-based diagonal thresholds with a dynamic "Top N" approach for polyline visibility. - Updated `/api/v1/search/trails/cluster` to mark only the top `MAP_MAX_POLYLINES` (default 100) trails by bounding box diagonal as "large". - Modified `trails_search_bounding_box` store function to always fetch polylines for trails marked as large, regardless of zoom level. - Removed obsolete `MAP_*_ZOOM_DIAGONAL_LIMIT` constants and simplified map layer constructors by removing tier-based filtering. - Updated environment configurations and documentation to use the new `PUBLIC_MAP_MAX_POLYLINES` variable. * refactor: remove obsolete zoom-based map clustering thresholds Following the transition to dynamic server-side clustering based on result density (Top N polylines), this commit removes all remaining zoom-based thresholds and logic. - Removed PUBLIC_MAP_LOW_ZOOM_THRESHOLD, PUBLIC_MAP_MEDIUM_ZOOM_THRESHOLD, and PUBLIC_MAP_HIGH_ZOOM_THRESHOLD environment variables. - Removed the 'mapClusterMinZoom' user setting from the schema, models, and settings UI. - Simplified the map component and MapLibre layer managers by removing unused 'clusterMinZoom', 'minZoom', and 'maxZoom' parameters. - Cleaned up obsolete 'map-cluster-zoom-level' translation keys across all locales. - Updated documentation to reflect the removal of these variables. * chore: remove map clustering debug logs * docs: update changelog and fix global zoom feature disappearance * fix: address PR reviews and refine map cluster/preview visuals * feat: implement configurable map cluster zoom and trail start marker settings * feat: render single unclustered shedded trails as start markers instead of cluster circles * minor fixes * minor ui changes * fix missing entries in map list * remove redundant map clustering zoom clamp * fix merge issues * fix merge issues * fixes * hide popup for trails without details * several improvements * prevent import of client trail_store in server cluster code * update changelog --------- Co-authored-by: slothful-vassal <89943360+slothful-vassal@users.noreply.github.com>
2026-06-07 13:00:26 +02:00
parent fc735130cc
commit 197cd8d04e
41 changed files with 1133 additions and 245 deletions
--- a/web/src/lib/stores/search_store.ts
+++ b/web/src/lib/stores/search_store.ts
@@ -99,7 +99,7 @@ export async function searchTrails(q: string, options: SearchParams): Promise<Hi

    const response: SearchResponse<TrailSearchResult> = await r.json();

-    return response.hits
+    return response.hits || []
 }

 export async function searchLocations(q: string, limit?: number, f: (url: RequestInfo | URL, config?: RequestInit) => Promise<Response> = fetch): Promise<Hits<LocationSearchResult>> {
@@ -245,6 +245,10 @@ export async function searchMulti(options: MultiSearchParams): Promise<MultiSear

    const response: MultiSearchResponse<any> = await r.json();

+    if (!response.results) {
+        return [];
+    }
+

    if (locationQuery && locationQuery.q !== undefined && locationQuery.q !== null) {
        const locationsResults = await searchLocations(locationQuery.q, locationQuery.limit)
--- a/web/src/lib/stores/trail_store.ts
+++ b/web/src/lib/stores/trail_store.ts
@@ -1,5 +1,6 @@
 import type { SummitLog } from "$lib/models/summit_log";
 import type { Tag } from "$lib/models/tag";
+import { MAP_MAX_POLYLINES } from "$lib/config/map";
 import { defaultTrailSearchAttributes, Trail, type TrailFilter, type TrailFilterValues, type TrailSearchResult } from "$lib/models/trail";
 import type { Waypoint } from "$lib/models/waypoint";
 import { APIError } from "$lib/util/api_util";
@@ -14,11 +15,6 @@ import { tags_create } from "./tag_store";
 import { currentUser } from "./user_store";
 import { waypoints_create, waypoints_delete, waypoints_update } from "./waypoint_store";

-let trails: Trail[] = []
-export const trail: Writable<Trail> = writable(new Trail(""));
-
-export const editTrail: Writable<Trail> = writable(new Trail(""));
-
 export async function trails_index(perPage: number = 21, random: boolean = false, f: (url: RequestInfo | URL, config?: RequestInit) => Promise<Response> = fetch) {
    const r = await f('/api/v1/trail?' + new URLSearchParams({
        "perPage": perPage.toString(),
@@ -93,7 +89,50 @@ export async function trails_search_filter(filter: TrailFilter, page: number = 1

 }

-export async function trails_search_bounding_box(northEast: M.LngLat, southWest: M.LngLat, filter: TrailFilter, page: number = 1, includePolyline: boolean = true) {
+const DETAILED_CACHE_MAX_SIZE = Math.max(200, MAP_MAX_POLYLINES * 10);
+
+let trails: Trail[] = []
+const detailedCache = new Map<string, Trail>();
+let detailedCacheKey = "";
+
+function getDetailedCache(id: string): Trail | undefined {
+    const cached = detailedCache.get(id);
+    if (!cached) {
+        return undefined;
+    }
+
+    detailedCache.delete(id);
+    // Reinsert the entry so Map iteration order tracks recent usage for LRU eviction.
+    detailedCache.set(id, cached);
+    return cached;
+}
+
+function setDetailedCache(id: string, trail: Trail) {
+    detailedCache.delete(id);
+    detailedCache.set(id, trail);
+
+    while (detailedCache.size > DETAILED_CACHE_MAX_SIZE) {
+        const oldestKey = detailedCache.keys().next().value;
+        if (!oldestKey) {
+            break;
+        }
+        detailedCache.delete(oldestKey);
+    }
+}
+
+export const trail: Writable<Trail> = writable(new Trail(""));
+
+export const editTrail: Writable<Trail> = writable(new Trail(""));
+
+export async function trails_search_bounding_box(
+    northEast: M.LngLat,
+    southWest: M.LngLat,
+    filter: TrailFilter,
+    page: number = 1,
+    zoom: number = 11,
+    perPage: number = 50,
+    loadMapData: boolean = true
+) {
    const user = get(currentUser)

    let filterText: string = "";
@@ -102,36 +141,182 @@ export async function trails_search_bounding_box(northEast: M.LngLat, southWest:
        filterText = buildFilterText(user, filter, false);
    }

-    let r = await fetch("/api/v1/search/trails", {
-        method: "POST",
-        body: JSON.stringify({
-            q: "",
-            options: {
-                filter: [
-                    `_geoBoundingBox([${northEast.lat}, ${northEast.lng}], [${southWest.lat}, ${southWest.lng}])`,
-                    filterText
-                ],
-                sort: [`${filter.sort}:${filter.sortOrder == "+" ? "asc" : "desc"}`,],
-                attributesToRetrieve: [...defaultTrailSearchAttributes, ...(includePolyline ? ["polyline"] : [])],
-                hitsPerPage: 500,
-                page: page
-            }
-        }),
-    });
-    const result: { page: number, totalPages: number, hits: Hits<TrailSearchResult> } = await r.json();
-
-    if (result.hits.length == 0) {
-        trails = [];
-        return { trails: [], ...result }
+    let lonFilter = `max_lon >= ${southWest.lng} AND min_lon <= ${northEast.lng}`;
+    if (southWest.lng > northEast.lng) {
+        lonFilter = `(max_lon >= ${southWest.lng} OR min_lon <= ${northEast.lng})`;
    }

-    const resultTrails: Trail[] = await searchResultToTrailList(result.hits)
+    const geoFilter = `max_lat >= ${southWest.lat} AND min_lat <= ${northEast.lat} AND ${lonFilter}`;
+    const listFilter = [filterText, geoFilter].filter(Boolean).join(" AND ");
+    const cacheKey = JSON.stringify({
+        q: filter.q,
+        filterText,
+        sort: filter.sort,
+        sortOrder: filter.sortOrder,
+    });
+    if (cacheKey !== detailedCacheKey) {
+        detailedCache.clear();
+        detailedCacheKey = cacheKey;
+    }

-    trails = page > 1 ? trails.concat(resultTrails) : resultTrails
+    // Step 1: Fetch paginated trails for the side list.
+    const listResponse = await fetch("/api/v1/search/trails", {
+        method: "POST",
+        body: JSON.stringify({
+            q: filter.q,
+            options: {
+                filter: listFilter,
+                attributesToRetrieve: defaultTrailSearchAttributes,
+                sort: [`${filter.sort}:${filter.sortOrder == "+" ? "asc" : "desc"}`],
+                hitsPerPage: perPage,
+                page,
+            },
+        }),
+    });

-    return { trails, ...result };
+    if (!listResponse.ok) {
+        const response = await listResponse.json();
+        throw new APIError(listResponse.status, response.message, response.detail)
+    }

+    const listResult: { page: number, totalPages: number, totalHits?: number, estimatedTotalHits?: number, hits: Hits<TrailSearchResult> } = await listResponse.json();
+    const listTrails = listResult.hits.length > 0
+        ? await searchResultToTrailList(listResult.hits)
+        : [];

+    trails = page > 1 ? trails.concat(listTrails) : listTrails;
+
+    if (!loadMapData) {
+        return {
+            trails,
+            mapTrails: [],
+            clusters: undefined,
+            estimatedTotalHits: listResult.estimatedTotalHits,
+            totalHits: listResult.totalHits ?? listResult.estimatedTotalHits,
+            totalPages: listResult.totalPages
+        };
+    }
+
+    // Step 2: Fetch server-side clusters and unclustered points for the map.
+    let cr = await fetch("/api/v1/search/trails/cluster", {
+        method: "POST",
+        body: JSON.stringify({
+            southWest: { lat: southWest.lat, lng: southWest.lng },
+            northEast: { lat: northEast.lat, lng: northEast.lng },
+            zoom,
+            q: filter.q,
+            filterText
+        })
+    });
+
+    if (!cr.ok) {
+        const response = await cr.json();
+        throw new APIError(cr.status, response.message, response.detail)
+    }
+
+    const clusterResult = await cr.json();
+    const clusterFeatureCollection = clusterResult;
+
+    const unclusteredFeatures = clusterFeatureCollection.features
+        .filter((f: any) => !f.properties.cluster);
+
+    // Extract IDs of visible unclustered points that are large enough to show details for
+    const unclusteredIds = unclusteredFeatures
+        .filter((f: any) => f.properties.is_large)
+        .map((f: any) => f.properties.id);
+
+    // Step 3: Identify which visible trails are MISSING from the local cache
+    const missingIds = unclusteredIds.filter((id: string) => !detailedCache.has(id));
+
+    // Step 4: Only fetch details for missing trails
+    if (missingIds.length > 0) {
+        const batchSize = 100; // Meilisearch filter length safety
+        for (let i = 0; i < missingIds.length; i += batchSize) {
+            const batch = missingIds.slice(i, i + batchSize);
+            const detailBatchQuery = {
+                indexUid: "trails",
+                q: "",
+                filter: [`id IN [${batch.map((id: string) => `'${id}'`).join(",")}]`],
+                attributesToRetrieve: [...defaultTrailSearchAttributes, "polyline"],
+                hitsPerPage: batchSize,
+            };
+
+            const dr = await fetch("/api/v1/search/multi", {
+                method: "POST",
+                body: JSON.stringify({ queries: [detailBatchQuery] }),
+            });
+
+            if (dr.ok) {
+                const detailResult = await dr.json();
+                const newTrails = await searchResultToTrailList(detailResult.results[0].hits);
+                // Populate cache
+                newTrails.forEach(t => {
+                    if (t.id) setDetailedCache(t.id, t)
+                });
+            }
+        }
+    }
+
+    // Step 5: Convert unclustered hits to lightweight Trail objects for map popups/previews.
+    const mapTrails: Trail[] = unclusteredFeatures
+        .map((f: any) => {
+        const s = f.properties;
+        const lat = f.geometry.coordinates[1];
+        const lng = f.geometry.coordinates[0];
+
+        const cached = getDetailedCache(s.id);
+        if (cached) {
+            return {
+                ...cached,
+                lat,
+                lon: lng,
+                bounding_box_diagonal: s.bounding_box_diagonal ?? cached.bounding_box_diagonal,
+                // Strip polyline for small trails so they don't linger as lines when zoomed out
+                polyline: s.is_large ? cached.polyline : undefined,
+            };
+        }
+
+        // Lightweight fallback for map markers
+        const t: Trail & RecordModel = {
+            id: s.id,
+            lat: lat,
+            lon: lng,
+            name: "",
+            author: "",
+            photos: [],
+            public: true,
+            completed: false,
+            summit_logs: [],
+            waypoints: [],
+            tags: [],
+            category: "",
+            created: new Date(0).toISOString(),
+            date: new Date(0).toISOString(),
+            updated: new Date(0).toISOString(),
+            description: "",
+            difficulty: "easy",
+            distance: 0,
+            duration: 0,
+            elevation_gain: 0,
+            elevation_loss: 0,
+            location: "",
+            bounding_box_diagonal: s.bounding_box_diagonal ?? 0,
+            like_count: 0,
+            collectionId: "trails",
+            collectionName: "trails",
+            expand: { author: {} as any }
+        };
+        return t;
+    });
+
+    return {
+        trails,
+        mapTrails,
+        clusters: clusterFeatureCollection,
+        estimatedTotalHits: listResult.estimatedTotalHits ?? clusterResult.totalHits,
+        totalHits: listResult.totalHits ?? listResult.estimatedTotalHits ?? clusterResult.totalHits,
+        totalPages: listResult.totalPages
+    };
 }

 export async function trails_show(id: string, handle?: string, share?: string, loadGPX?: boolean, f: (url: RequestInfo | URL, config?: RequestInit) => Promise<Response> = fetch) {
@@ -503,10 +688,12 @@ export async function fetchGPX(trail: { gpx?: string } & Record<string, any>, f:
 export async function searchResultToTrailList(hits: Hits<TrailSearchResult>): Promise<Trail[]> {
    const trails: Trail[] = []
    for (const h of hits) {
+        const created = Number(h.created || 0);
+        const date = Number(h.date || 0);
        const t: Trail & RecordModel = {
            collectionId: "trails",
            collectionName: "trails",
-            updated: new Date(h.created * 1000).toISOString(),
+            updated: new Date(created * 1000).toISOString(),
            author: h.author_name,
            name: h.name,
            photos: h.thumbnail ? [h.thumbnail] : [],
@@ -516,8 +703,8 @@ export async function searchResultToTrailList(hits: Hits<TrailSearchResult>): Pr
            waypoints: [],
            tags: h.tags ?? [],
            category: h.category,
-            created: new Date(h.created * 1000).toISOString(),
-            date: new Date(h.date * 1000).toISOString(),
+            created: new Date(created * 1000).toISOString(),
+            date: new Date(date * 1000).toISOString(),
            description: h.description,
            difficulty: h.difficulty == 0 ? "easy" : h.difficulty == 1 ? "moderate" : "difficult",
            distance: h.distance,
@@ -530,6 +717,7 @@ export async function searchResultToTrailList(hits: Hits<TrailSearchResult>): Pr
            location: h.location,
            gpx: h.gpx,
            polyline: h.polyline,
+            bounding_box_diagonal: h.bounding_box_diagonal ?? 0,
            domain: h.domain,
            iri: h.iri,
            thumbnail: 0,