server sided external api calls (#697)

* server sided external api calls

* fix uploading gpx files

* rename api/v1/nominatim with api/v1/geocoding to prepare photon support

* remove NOMINATIM_MAX_RETRIES (side-kick from other PR)

* re-add nominatim url fallback

* docu, env var defaults and fallbacks, docker compose files

* refactor

* furhter refactorings and fixes

* PUBLIC_VALHALLA_ENABLED in docker compose files added

* fix env var names

---------

Co-authored-by: Flomp <Flomp@users.noreply.github.com>

web/src/lib/server/http.ts

@@ -0,0 +1,18 @@
+import { json } from "@sveltejs/kit";
+
+function safeJson(text: string): any {
+    try {
+        return JSON.parse(text);
+    } catch {
+        return { message: text };
+    }
+}
+
+export async function proxyJsonResponse(response: Response) {
+    const text = await response.text();
+    const payload = text.length ? safeJson(text) : {};
+    if (!response.ok) {
+        return json(payload, { status: response.status });
+    }
+    return json(payload);
+}

web/src/lib/server/nominatim.ts

@@ -0,0 +1,63 @@
+import { version } from "$app/environment";
+import { resolveBaseUrl } from "$lib/server/url";
+import type { RequestEvent } from "@sveltejs/kit";
+
+const NOMINATIM_RATE_LIMIT_MS = 1000;
+const NOMINATIM_MAX_RETRIES = 2;
+let lastNominatimCall = 0;
+
+function getNominatimBaseUrl(): string {
+    return resolveBaseUrl("NOMINATIM_URL", "https://nominatim.openstreetmap.org");
+}
+
+function needsRateLimiting(baseUrl: string): boolean {
+    return baseUrl.includes("nominatim.openstreetmap.org");
+}
+
+const waitTimer = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
+
+async function nominatimRateLimiter(baseUrl: string) {
+    if (!needsRateLimiting(baseUrl)) {
+        return;
+    }
+
+    const elapsedTimeMs = Date.now() - lastNominatimCall;
+    const waitTime = NOMINATIM_RATE_LIMIT_MS - elapsedTimeMs;
+    if (waitTime > 0) {
+        await waitTimer(waitTime);
+    }
+
+    lastNominatimCall = Date.now();
+}
+
+export async function fetchNominatim(event: RequestEvent, path: string, params: URLSearchParams): Promise<Response> {
+    const baseUrl = getNominatimBaseUrl();
+    const base = new URL(baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`);
+    const cleanPath = path.replace(/^\/+/, "");
+    const url = new URL(cleanPath, base);
+    const query = params.toString();
+    if (query.length) {
+        url.search = query;
+    }
+
+    let attempt = 0;
+
+    while (true) {
+        await nominatimRateLimiter(baseUrl);
+
+        try {
+            return await event.fetch(url.toString(), {
+                method: "GET",
+                headers: {
+                    "User-Agent": `wanderer/${version}`,
+                },
+            });
+        } catch (error) {
+            if (attempt < NOMINATIM_MAX_RETRIES) {
+                attempt++;
+                continue;
+            }
+            throw new Error(`Nominatim fetch failed for ${url.toString()}`, { cause: error });
+        }
+    }
+}

web/src/lib/server/overpass.ts

@@ -0,0 +1,34 @@
+import { resolveBaseUrl } from "$lib/server/url";
+import type { RequestEvent } from "@sveltejs/kit";
+
+const OVERPASS_MAX_RETRIES = 2;
+
+function getOverpassBaseUrl(): string {
+    return resolveBaseUrl("OVERPASS_API_URL", "https://overpass-api.de");
+}
+
+export async function fetchOverpass(event: RequestEvent, params: URLSearchParams): Promise<Response> {
+    const baseUrl = getOverpassBaseUrl();
+    const base = new URL(baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`);
+    const url = new URL("api/interpreter", base);
+    const query = params.toString();
+    if (query.length) {
+        url.search = query;
+    }
+
+    let attempt = 0;
+
+    while (true) {
+        try {
+            return await event.fetch(url.toString(), {
+                method: "GET",
+            });
+        } catch (error) {
+            if (attempt < OVERPASS_MAX_RETRIES) {
+                attempt++;
+                continue;
+            }
+            throw error;
+        }
+    }
+}

web/src/lib/server/url.ts

@@ -0,0 +1,24 @@
+import { env as privateEnv } from "$env/dynamic/private";
+import { env as publicEnv } from "$env/dynamic/public";
+
+export type ExternalServiceUrlKey = "VALHALLA_URL" | "NOMINATIM_URL" | "OVERPASS_API_URL";
+
+export function normalizeBaseUrl(url: string): string {
+    const trimmedUrl = url.trim();
+    if (!trimmedUrl) {
+        return "";
+    }
+    if (!/^https?:\/\//i.test(trimmedUrl)) {
+        return `https://${trimmedUrl}`;
+    }
+    return trimmedUrl;
+}
+
+export function resolveBaseUrl(
+    key: ExternalServiceUrlKey,
+    fallback: string = "",
+): string {
+    const publicKey = `PUBLIC_${key}` as `PUBLIC_${string}`;
+    const rawUrl = privateEnv[key] ?? publicEnv[publicKey] ?? fallback;
+    return normalizeBaseUrl(rawUrl);
+}

web/src/lib/server/valhalla.ts

@@ -0,0 +1,5 @@
+import { resolveBaseUrl } from "$lib/server/url";
+
+export function getValhallaBaseUrl(): string {
+    return resolveBaseUrl("VALHALLA_URL");
+}

web/src/lib/stores/search_store.ts

@@ -1,10 +1,8 @@
-import { env } from "$env/dynamic/public";
 import type { Actor } from "$lib/models/activitypub/actor";
 import { defaultTrailSearchAttributes, type TrailSearchResult } from "$lib/models/trail";
 import { APIError } from "$lib/util/api_util";
 import type { Hits, MultiSearchParams, MultiSearchResponse, MultiSearchResult, SearchParams, SearchResponse } from "meilisearch";
 import type { ListResult } from "pocketbase";
-import { version } from "$app/environment";
 
 export type LocationSearchResult = {
     name: string;
@@ -104,14 +102,17 @@ export async function searchTrails(q: string, options: SearchParams): Promise<Hi
     return response.hits
 }
 
-export async function searchLocations(q: string, limit?: number): Promise<Hits<LocationSearchResult>> {
-    const nominatimURL = env.PUBLIC_NOMINATIM_URL ?? "https://nominatim.openstreetmap.org"
-    const r = await fetch(`${nominatimURL}/search?q=${q}&format=geojson&addressdetails=1${limit ? '&limit=' + limit : ''}`, {
-        method: "GET",
-        headers: new Headers({
-            "User-Agent": "wanderer/" + version
-        })
-    });
+export async function searchLocations(q: string, limit?: number, f: (url: RequestInfo | URL, config?: RequestInit) => Promise<Response> = fetch): Promise<Hits<LocationSearchResult>> {
+    if (!q.trim()) {
+        return [];
+    }
+
+    const params = new URLSearchParams({
+            q,
+            format: "geojson",
+            addressdetails: "1",
+        });
+    const r = await fetchGeocoding("search", params, f);
     if (!r.ok) {
         const response = await r.json();
         throw new APIError(r.status, response.message, response.detail)
@@ -127,14 +128,20 @@ export async function searchLocations(q: string, limit?: number): Promise<Hits<L
     }))
 }
 
-export async function searchLocationReverse(lat: number, lon: number) {
-    const nominatimURL = env.PUBLIC_NOMINATIM_URL ?? "https://nominatim.openstreetmap.org"
-    const r = await fetch(`${nominatimURL}/reverse?lat=${lat}&lon=${lon}&format=geojson&addressdetails=1`, {
-        method: "GET",
-        headers: new Headers({
-            "User-Agent": "wanderer/" + version
-        })
-    });
+async function fetchGeocoding(path: string, params: URLSearchParams, f: (url: RequestInfo | URL, config?: RequestInit) => Promise<Response> = fetch): Promise<Response> {
+    const query = params.toString();
+    const url = query.length ? `/api/v1/geocoding/${path}?${query}` : `/api/v1/geocoding/${path}`;
+    return await f(url);
+}
+
+export async function searchLocationReverse(lat: number, lon: number, f: (url: RequestInfo | URL, config?: RequestInit) => Promise<Response> = fetch) {
+    const params = new URLSearchParams({
+            lat: String(lat),
+            lon: String(lon),
+            format: "geojson",
+            addressdetails: "1",
+        });
+    const r = await fetchGeocoding("reverse", params, f);
     if (!r.ok) {
         const response = await r.json();
         throw new APIError(r.status, response.message, response.detail)
@@ -170,14 +177,17 @@ function getLocationDescription(address: Address) {
 
 export async function searchMulti(options: MultiSearchParams): Promise<MultiSearchResult<any>[]> {
 
-    const locationQuery = options.queries.find(q => q.indexUid === "locations");
-    const locationQueryIndex = locationQuery ? options.queries.indexOf(locationQuery) : -1
-    if (locationQueryIndex >= 0) {
-        options.queries.splice(locationQueryIndex, 1)
-    }
+    const locationQueryIndex = options.queries.findIndex(q => q.indexUid === "locations");
+    const locationQuery = locationQueryIndex >= 0 ? options.queries[locationQueryIndex] : undefined;
+    const queries = locationQueryIndex >= 0
+        ? options.queries.filter((_, index) => index !== locationQueryIndex)
+        : options.queries;
     const r = await fetch("/api/v1/search/multi", {
         method: "POST",
-        body: JSON.stringify(options),
+        body: JSON.stringify({
+            ...options,
+            queries,
+        }),
     });
 
     if (!r.ok) {

web/src/lib/vendor/maplibre-layer-manager/overpass-layer.ts

@@ -12,7 +12,7 @@ import type { OverpassResponse } from "./types";
 import { env } from '$env/dynamic/public'
 
 export class OverpassLayer implements BaseLayer {
-    private overpassApiURL: string = (env.PUBLIC_OVERPASS_API_URL && env.PUBLIC_OVERPASS_API_URL.length > 0 ? env.PUBLIC_OVERPASS_API_URL : "https://overpass-api.de") + "/api/interpreter";
+    private overpassApiURL: string = "/api/v1/overpass/interpreter";
 
     data: GeoJSON.FeatureCollection = ({ type: 'FeatureCollection', features: [] });
 

web/src/routes/api/v1/geocoding/reverse/+server.ts

@@ -0,0 +1,36 @@
+import { json, type RequestEvent } from "@sveltejs/kit";
+import { proxyJsonResponse } from "$lib/server/http";
+import { fetchNominatim } from "$lib/server/nominatim";
+
+export async function GET(event: RequestEvent) {
+    const lat = event.url.searchParams.get("lat");
+    const lon = event.url.searchParams.get("lon");
+    if (!lat || !lon) {
+        return json({ message: "Missing query parameter: lat or lon" }, { status: 400 });
+    }
+
+    if (Number.isNaN(Number(lat)) || Number.isNaN(Number(lon))) {
+        return json({ message: "Invalid query parameter: lat or lon" }, { status: 400 });
+    }
+
+    const params = new URLSearchParams({
+        lat,
+        lon,
+        format: "geojson",
+        addressdetails: "1",
+    });
+
+    try {
+        const response = await fetchNominatim(event, "/reverse", params);
+        return await proxyJsonResponse(response);
+    } catch (error) {
+        const err = error instanceof Error ? error : new Error(String(error));
+        const detail = {
+            name: err.name,
+            message: err.message,
+            cause: err.cause instanceof Error ? err.cause.message : err.cause,
+        };
+        console.error("Nominatim reverse request failed", detail);
+        return json({ message: "Nominatim request failed", detail }, { status: 502 });
+    }
+}

web/src/routes/api/v1/geocoding/search/+server.ts

@@ -0,0 +1,38 @@
+import { json, type RequestEvent } from "@sveltejs/kit";
+import { proxyJsonResponse } from "$lib/server/http";
+import { fetchNominatim } from "$lib/server/nominatim";
+
+export async function GET(event: RequestEvent) {
+    const q = event.url.searchParams.get("q");
+    if (!q) {
+        return json({ message: "Missing query parameter: q" }, { status: 400 });
+    }
+
+    const limit = event.url.searchParams.get("limit");
+    if (limit !== null && Number.isNaN(Number(limit))) {
+        return json({ message: "Invalid query parameter: limit" }, { status: 400 });
+    }
+
+    const params = new URLSearchParams({
+        q,
+        format: "geojson",
+        addressdetails: "1",
+    });
+    if (limit) {
+        params.set("limit", limit);
+    }
+
+    try {
+        const response = await fetchNominatim(event, "/search", params);
+        return await proxyJsonResponse(response);
+    } catch (error) {
+        const err = error instanceof Error ? error : new Error(String(error));
+        const detail = {
+            name: err.name,
+            message: err.message,
+            cause: err.cause instanceof Error ? err.cause.message : err.cause,
+        };
+        console.error("Nominatim search request failed", detail);
+        return json({ message: "Nominatim request failed", detail }, { status: 502 });
+    }
+}

web/src/routes/api/v1/overpass/interpreter/+server.ts

@@ -0,0 +1,21 @@
+import { json, type RequestEvent } from "@sveltejs/kit";
+import { proxyJsonResponse } from "$lib/server/http";
+import { fetchOverpass } from "$lib/server/overpass";
+
+export async function GET(event: RequestEvent) {
+    const data = event.url.searchParams.get("data");
+    if (!data) {
+        return json({ message: "Missing query parameter: data" }, { status: 400 });
+    }
+
+    const params = new URLSearchParams({
+        data,
+    });
+
+    try {
+        const response = await fetchOverpass(event, params);
+        return await proxyJsonResponse(response);
+    } catch (error) {
+        return json({ message: "Overpass request failed" }, { status: 502 });
+    }
+}

web/src/routes/api/v1/trail/upload/+server.ts

@@ -75,8 +75,12 @@ export async function PUT(event: RequestEvent) {
         }
 
         if (trail.lat && trail.lon) {
-            const location = await searchLocationReverse(trail.lat, trail.lon)
-            trail.location ??= location;
+            try {
+                const location = await searchLocationReverse(trail.lat, trail.lon, event.fetch)
+                trail.location ??= location;
+            } catch (e: any) {
+                console.warn("Reverse geocoding failed during upload", e);
+            }
         }
 
         trail.public = event.locals.settings.privacy?.trails == "public"

web/src/routes/api/v1/valhalla/height/+server.ts

@@ -1,5 +1,6 @@
-import { env } from '$env/dynamic/public';
-import { error, json, type NumericRange, type RequestEvent } from "@sveltejs/kit";
+import { getValhallaBaseUrl } from '$lib/server/valhalla';
+import { proxyJsonResponse } from '$lib/server/http';
+import { json, type RequestEvent } from "@sveltejs/kit";
 
 
 /**
@@ -29,18 +30,15 @@ import { error, json, type NumericRange, type RequestEvent } from "@sveltejs/kit
  *         description: Internal Server Error
  */
 export async function POST(event: RequestEvent) {
+    const baseUrl = getValhallaBaseUrl();
     const data = await event.request.json()
-    if (!env.PUBLIC_VALHALLA_URL) {
-        return error(400, "PUBLIC_VALHALLA_URL not set")
+    if (!baseUrl) {
+        return json({ message: "VALHALLA_URL not set" }, { status: 400 })
     }
     try {
-        const r = await event.fetch(env.PUBLIC_VALHALLA_URL + '/height', { method: "POST", body: JSON.stringify(data) });        
-        const response = await r.json();
-        if (!r.ok) {
-            throw error(r.status as NumericRange<400,500>, response);
-        }
-        return json(response);
+        const response = await event.fetch(baseUrl + '/height', { method: "POST", body: JSON.stringify(data) });
+        return await proxyJsonResponse(response);
     } catch (e: any) {
-        throw error(e.status || 500, e)
+        return json({ message: "Valhalla request failed" }, { status: 502 })
     }
-}
+}

web/src/routes/api/v1/valhalla/route/+server.ts

@@ -1,6 +1,10 @@
-import { env } from '$env/dynamic/public';
-import { error, json, type NumericRange, type RequestEvent } from "@sveltejs/kit";
+import { getValhallaBaseUrl } from '$lib/server/valhalla';
+import { proxyJsonResponse } from '$lib/server/http';
+import { json, type RequestEvent } from "@sveltejs/kit";
 
+type RouteRequestBody = Record<string, unknown> & {
+    include_elevation_profile?: boolean;
+};
 
 /**
  * @swagger
@@ -29,19 +33,19 @@ import { error, json, type NumericRange, type RequestEvent } from "@sveltejs/kit
  *         description: Internal Server Error
  */
 export async function POST(event: RequestEvent) {
-    const data = await event.request.json()
-    if (!env.PUBLIC_VALHALLA_URL) {
-        return json({ message: "PUBLIC_VALHALLA_URL not set" }, { status: 400 })
+    const baseUrl = getValhallaBaseUrl();
+    const data: RouteRequestBody = await event.request.json();
+    if (!baseUrl) {
+        return json({ message: "VALHALLA_URL not set" }, { status: 400 })
     }
-    try {
-        const r = await event.fetch(env.PUBLIC_VALHALLA_URL + '/route', { method: "POST", body: JSON.stringify(data) });
-        const response = await r.json();
-        if (!r.ok) {
-            return json({ message: response }, { status: r.status })
 
-        }
-        return json(response);
+    try {
+        const response = await event.fetch(baseUrl + '/route', {
+            method: "POST",
+            body: JSON.stringify(data)
+        });
+        return await proxyJsonResponse(response);
     } catch (e: any) {
-        return json({ message: e }, { status: 500 })
+        return json({ message: "Valhalla request failed" }, { status: 502 })
     }
-}
+}

web/src/routes/trail/edit/[id]/+page.svelte

@@ -1,5 +1,4 @@
 <script lang="ts">
-    import { env } from "$env/dynamic/public";
     import Button from "$lib/components/base/button.svelte";
     import Datepicker from "$lib/components/base/datepicker.svelte";
     import Select from "$lib/components/base/select.svelte";
@@ -124,6 +123,13 @@
     let gpxFile: File | Blob | null = null;
 
     let drawingActive = $state(false);
+
+    function routeCalculationErrorText(error: unknown) {
+        if (error instanceof Error && error.message) {
+            return error.message;
+        }
+        return "Error calculating route";
+    }
     let overwriteGPX = false;
     let draggingMarker = false;
 
@@ -660,7 +666,7 @@
         } catch (e) {
             console.error(e);
             show_toast({
-                text: "Error calculating route",
+                text: routeCalculationErrorText(e),
                 icon: "close",
                 type: "error",
             });
@@ -832,7 +838,7 @@
         } catch (e) {
             console.error(e);
             show_toast({
-                text: "Error calculating route",
+                text: routeCalculationErrorText(e),
                 icon: "close",
                 type: "error",
             });
@@ -882,7 +888,7 @@
         } catch (e) {
             console.error(e);
             show_toast({
-                text: "Error calculating route",
+                text: routeCalculationErrorText(e),
                 icon: "close",
                 type: "error",
             });
@@ -1219,32 +1225,30 @@
                 ? $_("upload-new-file")
                 : $_("upload-file")}</Button
         >
-        {#if env.PUBLIC_VALHALLA_URL}
-            <div class="flex gap-4 items-center w-full">
-                <hr class="basis-full border-input-border" />
-                <span class="text-gray-500 uppercase">{$_("or")}</span>
-                <hr class="basis-full border-input-border" />
-            </div>
-            <button
-                class="btn-primary"
-                type="button"
-                onclick={async () => {
-                    if (drawingActive) {
-                        await stopDrawing();
-                    } else {
-                        startDrawing();
-                    }
-                }}
-            >
-                {$formData.expand?.gpx_data
-                    ? drawingActive
-                        ? $_("stop-editing")
-                        : $_("edit-route")
-                    : drawingActive
-                      ? $_("stop-drawing")
-                      : $_("draw-a-route")}</button
-            >
-        {/if}
+        <div class="flex gap-4 items-center w-full">
+            <hr class="basis-full border-input-border" />
+            <span class="text-gray-500 uppercase">{$_("or")}</span>
+            <hr class="basis-full border-input-border" />
+        </div>
+        <button
+            class="btn-primary"
+            type="button"
+            onclick={async () => {
+                if (drawingActive) {
+                    await stopDrawing();
+                } else {
+                    startDrawing();
+                }
+            }}
+        >
+            {$formData.expand?.gpx_data
+                ? drawingActive
+                    ? $_("stop-editing")
+                    : $_("edit-route")
+                : drawingActive
+                    ? $_("stop-drawing")
+                    : $_("draw-a-route")}</button
+        >
         <input
             type="file"
             name="gpx"