server sided external api calls (#697)

* server sided external api calls

* fix uploading gpx files

* rename api/v1/nominatim with api/v1/geocoding to prepare photon support

* remove NOMINATIM_MAX_RETRIES (side-kick from other PR)

* re-add nominatim url fallback

* docu, env var defaults and fallbacks, docker compose files

* refactor

* furhter refactorings and fixes

* PUBLIC_VALHALLA_ENABLED in docker compose files added

* fix env var names

---------

Co-authored-by: Flomp <Flomp@users.noreply.github.com>

web/src/lib/server/http.ts

@@ -0,0 +1,18 @@
+import { json } from "@sveltejs/kit";
+
+function safeJson(text: string): any {
+    try {
+        return JSON.parse(text);
+    } catch {
+        return { message: text };
+    }
+}
+
+export async function proxyJsonResponse(response: Response) {
+    const text = await response.text();
+    const payload = text.length ? safeJson(text) : {};
+    if (!response.ok) {
+        return json(payload, { status: response.status });
+    }
+    return json(payload);
+}

web/src/lib/server/nominatim.ts

@@ -0,0 +1,63 @@
+import { version } from "$app/environment";
+import { resolveBaseUrl } from "$lib/server/url";
+import type { RequestEvent } from "@sveltejs/kit";
+
+const NOMINATIM_RATE_LIMIT_MS = 1000;
+const NOMINATIM_MAX_RETRIES = 2;
+let lastNominatimCall = 0;
+
+function getNominatimBaseUrl(): string {
+    return resolveBaseUrl("NOMINATIM_URL", "https://nominatim.openstreetmap.org");
+}
+
+function needsRateLimiting(baseUrl: string): boolean {
+    return baseUrl.includes("nominatim.openstreetmap.org");
+}
+
+const waitTimer = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
+
+async function nominatimRateLimiter(baseUrl: string) {
+    if (!needsRateLimiting(baseUrl)) {
+        return;
+    }
+
+    const elapsedTimeMs = Date.now() - lastNominatimCall;
+    const waitTime = NOMINATIM_RATE_LIMIT_MS - elapsedTimeMs;
+    if (waitTime > 0) {
+        await waitTimer(waitTime);
+    }
+
+    lastNominatimCall = Date.now();
+}
+
+export async function fetchNominatim(event: RequestEvent, path: string, params: URLSearchParams): Promise<Response> {
+    const baseUrl = getNominatimBaseUrl();
+    const base = new URL(baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`);
+    const cleanPath = path.replace(/^\/+/, "");
+    const url = new URL(cleanPath, base);
+    const query = params.toString();
+    if (query.length) {
+        url.search = query;
+    }
+
+    let attempt = 0;
+
+    while (true) {
+        await nominatimRateLimiter(baseUrl);
+
+        try {
+            return await event.fetch(url.toString(), {
+                method: "GET",
+                headers: {
+                    "User-Agent": `wanderer/${version}`,
+                },
+            });
+        } catch (error) {
+            if (attempt < NOMINATIM_MAX_RETRIES) {
+                attempt++;
+                continue;
+            }
+            throw new Error(`Nominatim fetch failed for ${url.toString()}`, { cause: error });
+        }
+    }
+}

web/src/lib/server/overpass.ts

@@ -0,0 +1,34 @@
+import { resolveBaseUrl } from "$lib/server/url";
+import type { RequestEvent } from "@sveltejs/kit";
+
+const OVERPASS_MAX_RETRIES = 2;
+
+function getOverpassBaseUrl(): string {
+    return resolveBaseUrl("OVERPASS_API_URL", "https://overpass-api.de");
+}
+
+export async function fetchOverpass(event: RequestEvent, params: URLSearchParams): Promise<Response> {
+    const baseUrl = getOverpassBaseUrl();
+    const base = new URL(baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`);
+    const url = new URL("api/interpreter", base);
+    const query = params.toString();
+    if (query.length) {
+        url.search = query;
+    }
+
+    let attempt = 0;
+
+    while (true) {
+        try {
+            return await event.fetch(url.toString(), {
+                method: "GET",
+            });
+        } catch (error) {
+            if (attempt < OVERPASS_MAX_RETRIES) {
+                attempt++;
+                continue;
+            }
+            throw error;
+        }
+    }
+}

web/src/lib/server/url.ts

@@ -0,0 +1,24 @@
+import { env as privateEnv } from "$env/dynamic/private";
+import { env as publicEnv } from "$env/dynamic/public";
+
+export type ExternalServiceUrlKey = "VALHALLA_URL" | "NOMINATIM_URL" | "OVERPASS_API_URL";
+
+export function normalizeBaseUrl(url: string): string {
+    const trimmedUrl = url.trim();
+    if (!trimmedUrl) {
+        return "";
+    }
+    if (!/^https?:\/\//i.test(trimmedUrl)) {
+        return `https://${trimmedUrl}`;
+    }
+    return trimmedUrl;
+}
+
+export function resolveBaseUrl(
+    key: ExternalServiceUrlKey,
+    fallback: string = "",
+): string {
+    const publicKey = `PUBLIC_${key}` as `PUBLIC_${string}`;
+    const rawUrl = privateEnv[key] ?? publicEnv[publicKey] ?? fallback;
+    return normalizeBaseUrl(rawUrl);
+}

web/src/lib/server/valhalla.ts

@@ -0,0 +1,5 @@
+import { resolveBaseUrl } from "$lib/server/url";
+
+export function getValhallaBaseUrl(): string {
+    return resolveBaseUrl("VALHALLA_URL");
+}

web/src/lib/stores/search_store.ts

@@ -1,10 +1,8 @@
-import { env } from "$env/dynamic/public";
 import type { Actor } from "$lib/models/activitypub/actor";
 import { defaultTrailSearchAttributes, type TrailSearchResult } from "$lib/models/trail";
 import { APIError } from "$lib/util/api_util";
 import type { Hits, MultiSearchParams, MultiSearchResponse, MultiSearchResult, SearchParams, SearchResponse } from "meilisearch";
 import type { ListResult } from "pocketbase";
-import { version } from "$app/environment";
 
 export type LocationSearchResult = {
     name: string;
@@ -104,14 +102,17 @@ export async function searchTrails(q: string, options: SearchParams): Promise<Hi
     return response.hits
 }
 
-export async function searchLocations(q: string, limit?: number): Promise<Hits<LocationSearchResult>> {
-    const nominatimURL = env.PUBLIC_NOMINATIM_URL ?? "https://nominatim.openstreetmap.org"
-    const r = await fetch(`${nominatimURL}/search?q=${q}&format=geojson&addressdetails=1${limit ? '&limit=' + limit : ''}`, {
-        method: "GET",
-        headers: new Headers({
-            "User-Agent": "wanderer/" + version
-        })
-    });
+export async function searchLocations(q: string, limit?: number, f: (url: RequestInfo | URL, config?: RequestInit) => Promise<Response> = fetch): Promise<Hits<LocationSearchResult>> {
+    if (!q.trim()) {
+        return [];
+    }
+
+    const params = new URLSearchParams({
+            q,
+            format: "geojson",
+            addressdetails: "1",
+        });
+    const r = await fetchGeocoding("search", params, f);
     if (!r.ok) {
         const response = await r.json();
         throw new APIError(r.status, response.message, response.detail)
@@ -127,14 +128,20 @@ export async function searchLocations(q: string, limit?: number): Promise<Hits<L
     }))
 }
 
-export async function searchLocationReverse(lat: number, lon: number) {
-    const nominatimURL = env.PUBLIC_NOMINATIM_URL ?? "https://nominatim.openstreetmap.org"
-    const r = await fetch(`${nominatimURL}/reverse?lat=${lat}&lon=${lon}&format=geojson&addressdetails=1`, {
-        method: "GET",
-        headers: new Headers({
-            "User-Agent": "wanderer/" + version
-        })
-    });
+async function fetchGeocoding(path: string, params: URLSearchParams, f: (url: RequestInfo | URL, config?: RequestInit) => Promise<Response> = fetch): Promise<Response> {
+    const query = params.toString();
+    const url = query.length ? `/api/v1/geocoding/${path}?${query}` : `/api/v1/geocoding/${path}`;
+    return await f(url);
+}
+
+export async function searchLocationReverse(lat: number, lon: number, f: (url: RequestInfo | URL, config?: RequestInit) => Promise<Response> = fetch) {
+    const params = new URLSearchParams({
+            lat: String(lat),
+            lon: String(lon),
+            format: "geojson",
+            addressdetails: "1",
+        });
+    const r = await fetchGeocoding("reverse", params, f);
     if (!r.ok) {
         const response = await r.json();
         throw new APIError(r.status, response.message, response.detail)
@@ -170,14 +177,17 @@ function getLocationDescription(address: Address) {
 
 export async function searchMulti(options: MultiSearchParams): Promise<MultiSearchResult<any>[]> {
 
-    const locationQuery = options.queries.find(q => q.indexUid === "locations");
-    const locationQueryIndex = locationQuery ? options.queries.indexOf(locationQuery) : -1
-    if (locationQueryIndex >= 0) {
-        options.queries.splice(locationQueryIndex, 1)
-    }
+    const locationQueryIndex = options.queries.findIndex(q => q.indexUid === "locations");
+    const locationQuery = locationQueryIndex >= 0 ? options.queries[locationQueryIndex] : undefined;
+    const queries = locationQueryIndex >= 0
+        ? options.queries.filter((_, index) => index !== locationQueryIndex)
+        : options.queries;
     const r = await fetch("/api/v1/search/multi", {
         method: "POST",
-        body: JSON.stringify(options),
+        body: JSON.stringify({
+            ...options,
+            queries,
+        }),
     });
 
     if (!r.ok) {

web/src/lib/vendor/maplibre-layer-manager/overpass-layer.ts

@@ -12,7 +12,7 @@ import type { OverpassResponse } from "./types";
 import { env } from '$env/dynamic/public'
 
 export class OverpassLayer implements BaseLayer {
-    private overpassApiURL: string = (env.PUBLIC_OVERPASS_API_URL && env.PUBLIC_OVERPASS_API_URL.length > 0 ? env.PUBLIC_OVERPASS_API_URL : "https://overpass-api.de") + "/api/interpreter";
+    private overpassApiURL: string = "/api/v1/overpass/interpreter";
 
     data: GeoJSON.FeatureCollection = ({ type: 'FeatureCollection', features: [] });