From 550e3daa43a933c1b34b965648e0cbd111339a61 Mon Sep 17 00:00:00 2001 From: Christian Beutel <> Date: Mon, 10 Feb 2025 18:10:14 +0100 Subject: [PATCH] map speed improvements --- db/main.go | 27 +++- db/util/meilisearch.go | 20 ++- .../lib/components/trail/trail_card.svelte | 6 +- .../components/trail/trail_info_panel.svelte | 6 +- .../components/trail/trail_timeline.svelte | 2 +- web/src/lib/models/trail.ts | 37 ++++- web/src/lib/stores/trail_store.ts | 146 ++++++++++-------- web/src/lib/util/maplibre_util.ts | 2 +- web/src/routes/api/v1/trail/+server.ts | 3 +- web/src/routes/map/+page.svelte | 52 ++++++- 10 files changed, 207 insertions(+), 94 deletions(-) diff --git a/db/main.go b/db/main.go index 2b42f05c..fb4eaa4b 100644 --- a/db/main.go +++ b/db/main.go @@ -59,10 +59,10 @@ func registerMigrations(app *pocketbase.PocketBase) { func setupEventHandlers(app *pocketbase.PocketBase, client meilisearch.ServiceManager) { app.OnModelAfterCreate("users").Add(createUserHandler(app, client)) - app.OnModelAfterCreate("trails").Add(createTrailIndexHandler(client)) + app.OnModelAfterCreate("trails").Add(createTrailIndexHandler(app, client)) app.OnRecordAfterCreateRequest("trails").Add(createTrailHandler(app)) - app.OnRecordAfterUpdateRequest("trails").Add(updateTrailHandler(client)) + app.OnRecordAfterUpdateRequest("trails").Add(updateTrailHandler(app, client)) app.OnRecordAfterDeleteRequest("trails").Add(deleteTrailHandler(client)) app.OnRecordAfterCreateRequest("trail_share").Add(createTrailShareHandler(app, client)) @@ -128,10 +128,14 @@ func createDefaultUserSettings(app *pocketbase.PocketBase, userId string) error return app.Dao().SaveRecord(settings) } -func createTrailIndexHandler(client meilisearch.ServiceManager) func(e *core.ModelEvent) error { +func createTrailIndexHandler(app *pocketbase.PocketBase, client meilisearch.ServiceManager) func(e *core.ModelEvent) error { return func(e *core.ModelEvent) error { record := e.Model.(*models.Record) - if err := util.IndexTrail(record, client); err != nil { + author, err := app.Dao().FindRecordById("users", record.GetString(("author"))) + if err != nil { + return err + } + if err := util.IndexTrail(record, author, client); err != nil { return err } return nil @@ -156,9 +160,13 @@ func createTrailHandler(app *pocketbase.PocketBase) func(e *core.RecordCreateEve } } -func updateTrailHandler(client meilisearch.ServiceManager) func(e *core.RecordUpdateEvent) error { +func updateTrailHandler(app *pocketbase.PocketBase, client meilisearch.ServiceManager) func(e *core.RecordUpdateEvent) error { return func(e *core.RecordUpdateEvent) error { - return util.UpdateTrail(e.Record, client) + author, err := app.Dao().FindRecordById("users", e.Record.GetString(("author"))) + if err != nil { + return err + } + return util.UpdateTrail(e.Record, author, client) } } @@ -727,8 +735,13 @@ func bootstrapMeilisearchTrails(app *pocketbase.PocketBase, client meilisearch.S return err } + client.Index("trails").DeleteAllDocuments() for _, trail := range trails { - if err := util.UpdateTrail(trail, client); err != nil { + author, err := app.Dao().FindRecordById("users", trail.GetString(("author"))) + if err != nil { + return err + } + if err := util.IndexTrail(trail, author, client); err != nil { return err } } diff --git a/db/util/meilisearch.go b/db/util/meilisearch.go index 0b38ad87..678d2c6a 100644 --- a/db/util/meilisearch.go +++ b/db/util/meilisearch.go @@ -8,10 +8,18 @@ import ( "github.com/pocketbase/pocketbase/models" ) -func documentFromTrailRecord(r *models.Record, includeShares bool) map[string]interface{} { +func documentFromTrailRecord(r *models.Record, author *models.Record, includeShares bool) map[string]interface{} { + photos := r.GetStringSlice("photos") + thumbnail := "" + if len(photos) > 0 { + thumbnail = r.GetStringSlice("photos")[r.GetInt("thumbnail")] + } + document := map[string]interface{}{ "id": r.Id, "author": r.GetString("author"), + "author_name": author.GetString("username"), + "author_avatar": author.GetString("avatar"), "name": r.GetString("name"), "description": r.GetString("description"), "location": r.GetString("location"), @@ -25,6 +33,8 @@ func documentFromTrailRecord(r *models.Record, includeShares bool) map[string]in "date": r.GetDateTime("date").Time().Unix(), "created": r.GetDateTime("created").Time().Unix(), "public": r.GetBool("public"), + "thumbnail": thumbnail, + "gpx": r.GetString("gpx"), "_geo": map[string]float64{ "lat": r.GetFloat("lat"), "lng": r.GetFloat("lon"), @@ -56,8 +66,8 @@ func documentFromListRecord(r *models.Record, includeShares bool) map[string]int return document } -func IndexTrail(r *models.Record, client meilisearch.ServiceManager) error { - documents := []map[string]interface{}{documentFromTrailRecord(r, true)} +func IndexTrail(r *models.Record, author *models.Record, client meilisearch.ServiceManager) error { + documents := []map[string]interface{}{documentFromTrailRecord(r, author, true)} if _, err := client.Index("trails").AddDocuments(documents); err != nil { return err @@ -66,8 +76,8 @@ func IndexTrail(r *models.Record, client meilisearch.ServiceManager) error { return nil } -func UpdateTrail(r *models.Record, client meilisearch.ServiceManager) error { - documents := documentFromTrailRecord(r, false) +func UpdateTrail(r *models.Record, author *models.Record, client meilisearch.ServiceManager) error { + documents := documentFromTrailRecord(r, author, false) if _, err := client.Index("trails").UpdateDocuments(documents); err != nil { return err diff --git a/web/src/lib/components/trail/trail_card.svelte b/web/src/lib/components/trail/trail_card.svelte index 8dde82c6..ca98a4a3 100644 --- a/web/src/lib/components/trail/trail_card.svelte +++ b/web/src/lib/components/trail/trail_card.svelte @@ -55,7 +55,7 @@
- +
{#if (trail.public || trailIsShared) && pb.authStore.model}
{/if} {#if trail.expand?.trail_share_via_trail?.length} - + + + {/if}
{/if} diff --git a/web/src/lib/components/trail/trail_info_panel.svelte b/web/src/lib/components/trail/trail_info_panel.svelte index aeedf97f..191fbdc1 100644 --- a/web/src/lib/components/trail/trail_info_panel.svelte +++ b/web/src/lib/components/trail/trail_info_panel.svelte @@ -214,8 +214,8 @@
-
-

+
+

{trail.name}

{#if trail.date} @@ -370,7 +370,7 @@ {/if}

- {$_("route", { values: { n: 2 } })} + {$_("route", { values: { n: 1 } })}

{#if mode === "overview"}
{wp.name}
{wp.lat.toFixed(5)}, {wp.lon.toFixed(5)} {wp.lat.toFixed(5)}, {wp.lon.toFixed(5)}

{wp.description} diff --git a/web/src/lib/models/trail.ts b/web/src/lib/models/trail.ts index d4046fd0..3d2f5c76 100644 --- a/web/src/lib/models/trail.ts +++ b/web/src/lib/models/trail.ts @@ -27,10 +27,10 @@ class Trail { summit_logs: string[]; expand?: { category?: Category; - waypoints: Waypoint[] - summit_logs: SummitLog[] + waypoints?: Waypoint[] + summit_logs?: SummitLog[] author?: UserAnonymous - comments_via_trail: Comment[] + comments_via_trail?: Comment[] gpx_data?: string trail_share_via_trail?: TrailShare[] } @@ -142,6 +142,35 @@ interface TrailBoundingBox { min_lon: number, } + +interface TrailSearchResult { + id: string; + author: string; + author_name: string; + author_avatar: string; + name: string; + description: string; + location: string; + distance: number; + elevation_gain: number; + elevation_loss: number; + duration: number; + difficulty: "easy" | "moderate" | "difficult"; + category: string; + completed: boolean; + date: number; + created: number; + public: boolean; + thumbnail: string; + shares?: string[]; + gpx: string; + _geo: { + lat: number, + lng: number + }; +} + + export { Trail }; - export type { TrailBoundingBox, TrailFilter, TrailFilterValues }; +export type { TrailBoundingBox, TrailFilter, TrailFilterValues, TrailSearchResult }; diff --git a/web/src/lib/stores/trail_store.ts b/web/src/lib/stores/trail_store.ts index d53fa30b..f12de419 100644 --- a/web/src/lib/stores/trail_store.ts +++ b/web/src/lib/stores/trail_store.ts @@ -1,12 +1,12 @@ import type { SummitLog } from "$lib/models/summit_log"; -import { Trail, type TrailFilter, type TrailFilterValues } from "$lib/models/trail"; +import { Trail, type TrailFilter, type TrailFilterValues, type TrailSearchResult } from "$lib/models/trail"; import type { Waypoint } from "$lib/models/waypoint"; import { pb } from "$lib/pocketbase"; import { deepEqual } from "$lib/util/deep_util"; import { getFileURL } from "$lib/util/file_util"; import * as M from "maplibre-gl"; import type { Hits } from "meilisearch"; -import { type ListResult } from "pocketbase"; +import { type ListResult, type RecordModel } from "pocketbase"; import { writable, type Writable } from "svelte/store"; import { summit_logs_create, summit_logs_delete, summit_logs_update } from "./summit_log_store"; import { waypoints_create, waypoints_delete, waypoints_update } from "./waypoint_store"; @@ -59,7 +59,15 @@ export async function trails_search_filter(filter: TrailFilter, page: number = 1 let r = await f("/api/v1/search/trails", { method: "POST", - body: JSON.stringify({ q: filter.q, options: { filter: filterText, sort: [`${filter.sort}:${filter.sortOrder == "+" ? "asc" : "desc"}`], hitsPerPage: 12, page: page } }), + body: JSON.stringify({ + q: filter.q, + options: { + filter: filterText, + sort: [`${filter.sort}:${filter.sortOrder == "+" ? "asc" : "desc"}`], + hitsPerPage: 12, + page: page + } + }), }); if (!r.ok) { @@ -67,35 +75,19 @@ export async function trails_search_filter(filter: TrailFilter, page: number = 1 throw new APIError(r.status, response.message, response.detail) } - const result: { page: number, totalPages: number, hits: Hits> } = await r.json(); + const result: { page: number, totalPages: number, hits: Hits } = await r.json(); - - const trailIds = result.hits.map((h: Record) => h.id); - - if (trailIds.length == 0) { + if (result.hits.length == 0) { return { items: [], ...result }; } - r = await f('/api/v1/trail?' + new URLSearchParams({ - expand: "category,waypoints,summit_logs,trail_share_via_trail", - filter: `'${trailIds.join(',')}'~id`, - sort: `${filter.sortOrder}${filter.sort}` - }), { - method: 'GET', - }) + const resultTrails: Trail[] = await searchResultToTrailList(result.hits) - if (!r.ok) { - const response = await r.json(); - throw new APIError(r.status, response.message, response.detail) - } - - const response: ListResult = await r.json() - - return { items: response.items, ...result }; + return { items: resultTrails, ...result }; } -export async function trails_search_bounding_box(northEast: M.LngLat, southWest: M.LngLat, filter?: TrailFilter, loadGPX: boolean = true) { +export async function trails_search_bounding_box(northEast: M.LngLat, southWest: M.LngLat, filter?: TrailFilter, page: number = 1, loadGPX: boolean = true) { let filterText: string = ""; @@ -106,57 +98,29 @@ export async function trails_search_bounding_box(northEast: M.LngLat, southWest: let r = await fetch("/api/v1/search/trails", { method: "POST", body: JSON.stringify({ - q: "", options: { - limit: 100, + q: "", + options: { filter: [ `_geoBoundingBox([${northEast.lat}, ${northEast.lng}], [${southWest.lat}, ${southWest.lng}])`, filterText ], + hitsPerPage: 500, + page: page } }), }); - const result = await r.json(); + const result: { page: number, totalPages: number, hits: Hits } = await r.json(); - const trailIds = result.hits?.map((h: Record) => h.id) ?? []; - - if (trailIds.length == 0) { - const currentTrails: Trail[] = trails; - const comparison = compareObjectArrays(currentTrails, []); + if (result.hits.length == 0) { trails = []; - return { trails: [], ...comparison } + return { trails: [], ...result } } - r = await fetch('/api/v1/trail?' + new URLSearchParams({ - "perPage": "-1", - filter: `'${trailIds.join(',')}'~id`, - expand: "category,waypoints,summit_logs", - sort: `+name`, - }), { - method: 'GET', - }) + const resultTrails: Trail[] = await searchResultToTrailList(result.hits, loadGPX) - if (!r.ok) { - const response = await r.json(); - throw new APIError(r.status, response.message, response.detail) - } + trails = page > 1 ? trails.concat(resultTrails) : resultTrails - - const response = await r.json() - - if (loadGPX) { - for (const trail of response.items) { - const gpxData: string = await fetchGPX(trail); - if (!trail.expand) { - trail.expand = {}; - } - trail.expand.gpx_data = gpxData; - } - } - - const comparison = compareObjectArrays(trails, response.items) - trails = response.items; - - return { trails: response.items, ...comparison }; + return { trails, ...result }; } @@ -269,7 +233,7 @@ export async function trails_update(oldTrail: Trail, newTrail: Trail, photos?: F } for (const updatedWaypoint of waypointUpdates.updated) { - const oldWaypoint = oldTrail.expand?.waypoints.find(w => w.id == updatedWaypoint.id); + const oldWaypoint = oldTrail.expand?.waypoints?.find(w => w.id == updatedWaypoint.id); const model = await waypoints_update(oldWaypoint!, { ...updatedWaypoint, marker: undefined, @@ -288,7 +252,7 @@ export async function trails_update(oldTrail: Trail, newTrail: Trail, photos?: F } for (const updatedSummitLog of summitLogUpdates.updated) { - const oldSummitLog = oldTrail.expand?.summit_logs.find(w => w.id == updatedSummitLog.id); + const oldSummitLog = oldTrail.expand?.summit_logs?.find(w => w.id == updatedSummitLog.id); const model = await summit_logs_update(oldSummitLog!, updatedSummitLog); } @@ -448,6 +412,60 @@ export async function fetchGPX(trail: { gpx?: string } & Record, f: return gpxData } +async function searchResultToTrailList(hits: Hits, loadGPX: boolean = false): Promise { + const trails: Trail[] = [] + for (const h of hits) { + const t: Trail & RecordModel = { + collectionId: "trails", + collectionName: "trails", + updated: new Date(h.created * 1000).toISOString(), + author: h.author, + name: h.name, + photos: h.thumbnail ? [h.thumbnail] : [], + public: h.public, + summit_logs: [], + waypoints: [], + category: h.category, + created: new Date(h.created * 1000).toISOString(), + date: new Date(h.date * 1000).toISOString(), + description: h.description, + difficulty: h.difficulty, + distance: h.distance, + duration: h.duration, + elevation_gain: h.elevation_gain, + elevation_loss: h.elevation_loss, + id: h.id, + lat: h._geo.lat, + lon: h._geo.lng, + location: h.location, + gpx: h.gpx, + thumbnail: 0, + expand: { + author: { + collectionId: "users", + private: false, + id: h.author, + avatar: h.author_avatar, + username: h.author_name + } as any, + trail_share_via_trail: h.shares?.map(s => ({ + permission: "view", + trail: h.id, + user: s, + })), + } + } + + if (loadGPX) { + const gpxData: string = await fetchGPX(t); + t.expand!.gpx_data = gpxData; + } + + trails.push(t) + } + return trails +} + function buildFilterText(filter: TrailFilter, includeGeo: boolean): string { let filterText: string = ""; diff --git a/web/src/lib/util/maplibre_util.ts b/web/src/lib/util/maplibre_util.ts index dda05f78..31f9a173 100644 --- a/web/src/lib/util/maplibre_util.ts +++ b/web/src/lib/util/maplibre_util.ts @@ -158,7 +158,7 @@ export function createPopupFromTrail(trail: Trail) { // Create the image element const img = document.createElement("img"); - img.className = "h-full w-28 object-cover"; + img.className = "h-full w-20 object-cover"; img.src = thumbnail; // Set image source safely img.alt = ""; // Always include a safe alt attribute imageContainer.appendChild(img); diff --git a/web/src/routes/api/v1/trail/+server.ts b/web/src/routes/api/v1/trail/+server.ts index 9a7ccd3f..85a7f104 100644 --- a/web/src/routes/api/v1/trail/+server.ts +++ b/web/src/routes/api/v1/trail/+server.ts @@ -15,7 +15,8 @@ export async function GET(event: RequestEvent) { if (!t.expand) { t.expand = {} as any } - t.expand!.author = await pb.collection("users_anonymous").getOne(t.author); + + // t.expand!.author = await pb.collection("users_anonymous").getOne(t.author); t.expand?.waypoints?.sort((a, b) => (a.distance_from_start ?? 0) - (b.distance_from_start ?? 0)) } return json(r) diff --git a/web/src/routes/map/+page.svelte b/web/src/routes/map/+page.svelte index 9d66c58d..b73cf7d5 100644 --- a/web/src/routes/map/+page.svelte +++ b/web/src/routes/map/+page.svelte @@ -46,8 +46,12 @@ const MIN_ZOOM = 6; let loading: boolean = $state(true); + let loadingNextPage: boolean = false; - onMount(async () => {}); + let pagination = { + page: 1, + totalPages: 1, + }; async function search(q: string) { const r = await searchMulti({ @@ -103,15 +107,24 @@ } } - async function searchTrails(northEast: M.LngLat, southWest: M.LngLat) { - loading = true; - const changes = await trails_search_bounding_box( + async function searchTrails( + northEast: M.LngLat, + southWest: M.LngLat, + reset: boolean = true, + ) { + if (reset) { + pagination.page = 1; + loading = true; + } + const trailsInBox = await trails_search_bounding_box( northEast, southWest, filter, + pagination.page, (map?.getZoom() ?? 0) > MIN_ZOOM, ); - trails = changes.trails; + pagination.totalPages = trailsInBox.totalPages; + trails = trailsInBox.trails; loading = false; } @@ -147,7 +160,6 @@ bounds.getNorthEast().lat, ), }; - await searchTrails( normalizedBounds.northEast, normalizedBounds.southWest, @@ -214,6 +226,32 @@ ); } } + + async function onListScroll(e: Event) { + const container = e.target as HTMLDivElement; + const scrollTop = container.scrollTop; + const scrollHeight = container.scrollHeight; + const clientHeight = container.clientHeight; + + if ( + scrollTop + clientHeight >= scrollHeight * 0.8 && + pagination.page !== pagination.totalPages && + !loadingNextPage + ) { + loadingNextPage = true; + await loadNextPage(); + loadingNextPage = false; + } + } + + async function loadNextPage() { + if (!map) { + return; + } + pagination.page += 1; + const bounds = map.getBounds(); + await searchTrails(bounds.getNorthEast(), bounds.getSouthWest(), false); + } @@ -223,6 +261,7 @@