Feature: watcher based bulk uploads (#886)

* initial commit * fix watcher.js for docker * Update docs * update healthcheck * update docs --------- Co-authored-by: Christian Beutel <>
2026-04-20 13:46:32 +02:00
parent 1f17a8784b
commit 871bf38112
8 changed files with 155 additions and 126 deletions
--- a/web/Dockerfile
+++ b/web/Dockerfile
@@ -1,33 +1,3 @@
-FROM curlimages/curl:8.18.0 AS download-env
-
-# renovate: datasource=github-releases depName=stunnel/static-curl packageName=stunnel/static-curl
-ENV CURL_VERSION=8.18.0
-
-RUN set -eux ; \
-    ARCHITECTURE="$(uname -m)" ; \
-    case $ARCHITECTURE in \
-    x86_64) ARCHITECTURE="x86_64" ;; \
-    aarch64 | armv8* | arm64) ARCHITECTURE="aarch64" ;; \
-    *) \
-    echo "(!) Architecture $ARCHITECTURE unsupported" ; \
-    exit 1 \
-    ;; \
-    esac ; \
-    curl \
-    --connect-timeout 10 \
-    --fail \
-    --location \
-    --max-time 300 \
-    --output /tmp/curl.tar.xz \
-    --proto '=https' \
-    --show-error \
-    --silent \
-    --tlsv1.2 \
-    "https://github.com/stunnel/static-curl/releases/download/${CURL_VERSION}/curl-linux-${ARCHITECTURE}-glibc-${CURL_VERSION}.tar.xz" \
-    ; \
-    tar -xJf /tmp/curl.tar.xz -C /tmp ; \
-    chmod +x /tmp/curl ;
-
 FROM node:22-alpine AS build-env

 WORKDIR /app
@@ -45,12 +15,11 @@ FROM node:22-alpine
 WORKDIR /app/uploads
 WORKDIR /app

-COPY --from=download-env /tmp/curl /curl
 COPY --from=build-env /app /app
-COPY ./cron.sh /etc/periodic/15min/cron
-RUN chmod +x /etc/periodic/15min/cron
+RUN apk add --no-cache curl
+COPY watcher.js /app/watcher.js

-CMD crond && node build
+CMD ["npm", "run", "start"]

 EXPOSE 3000

--- a/web/cron.sh
+++ b/web/cron.sh
@@ -1,62 +0,0 @@
-#!/bin/ash
-
-# API endpoint URL
-API_URL="http://localhost:3000/api/v1"
-
-# Folder containing files to upload
-UPLOAD_FOLDER=$UPLOAD_FOLDER
-
-# Credentials for login
-USERNAME=$UPLOAD_USER
-PASSWORD=$UPLOAD_PASSWORD
-
-login() {
-    local username="$1"
-    local password="$2"
-
-    response=$(/curl -c cookie.txt --location --request POST "$API_URL/auth/login" --header 'Content-Type: application/json' --data-raw "{\"username\": \"$username\", \"password\": \"$password\"}")
-
-    # Check if login was successful (look for "200 OK" in response headers)
-    if [ $? -eq 0 ] && [ "$(echo "$response" | grep -c "token")" -eq 1 ]; then
-        echo "[INFO] [$(date +"%T")]: Login successful. Cookie obtained." > /proc/1/fd/1
-    else
-        echo "[ERROR] [$(date +"%T")]: Login failed. Unable to obtain cookie." > /proc/1/fd/1
-        exit 1
-    fi
-}
-
-# Function to upload file and delete if successful
-upload_and_delete() {
-    local file="$1"
-
-    ls "$file"
-    
-    # API call to upload file
-    response=$(/curl -b cookie.txt --location --request PUT "$API_URL/trail/upload" --header 'Content-Type: multipart/form-data' -F "file=@-" -F "name=$base_name" <"$file")
-    # Check if API call was successful (status code 200)
-    if [ $? -eq 0 ] && [ "$(echo "$response" | grep -c "author")" -eq 1 ]; then
-        echo "[INFO] [$(date +"%T")]: File $file uploaded successfully." > /proc/1/fd/1
-        # Delete the file
-        rm "$file"
-        echo "[INFO] [$(date +"%T")]: File $file deleted."
-    else
-        echo $response
-        echo "[ERROR] [$(date +"%T")]: Failed to upload file $file." > /proc/1/fd/1
-    fi
-}
-# Login to obtain cookie
-if [ -n "$USERNAME" ] && [ -n "$PASSWORD" ]; then
-    echo "[INFO] [$(date +"%T")]: Starting auto-upload" > /proc/1/fd/1
-
-    login "$USERNAME" "$PASSWORD"
-    # Iterate over each file in the folder
-    for file in "$UPLOAD_FOLDER"/*; do
-        # Check if file exists and is a regular file
-        if [ -f "$file" ]; then
-            upload_and_delete "$file"
-        fi
-    done
-
-    echo "[INFO] [$(date +"%T")]: Auto-upload completed" > /proc/1/fd/1
-
-fi
--- a/web/package-lock.json
+++ b/web/package-lock.json
@@ -36,6 +36,7 @@
        "chart.js": "^4.5.1",
        "chartjs-plugin-crosshair": "^2.0.0",
        "chartjs-plugin-zoom": "^2.1.0",
+        "chokidar": "^5.0.0",
        "crypto-random-string": "^5.0.0",
        "felte": "^1.3.0",
        "heic2any": "^0.0.4",
@@ -2857,16 +2858,15 @@
      }
    },
    "node_modules/chokidar": {
-      "version": "4.0.3",
-      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz",
-      "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==",
-      "dev": true,
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz",
+      "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==",
      "license": "MIT",
      "dependencies": {
-        "readdirp": "^4.0.1"
+        "readdirp": "^5.0.0"
      },
      "engines": {
-        "node": ">= 14.16.0"
+        "node": ">= 20.19.0"
      },
      "funding": {
        "url": "https://paulmillr.com/funding/"
@@ -5120,13 +5120,12 @@
      "license": "MIT"
    },
    "node_modules/readdirp": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.1.tgz",
-      "integrity": "sha512-h80JrZu/MHUZCyHu5ciuoI0+WxsCxzxJTILn6Fs8rxSnFPh+UVHYfeIxK1nVGugMqkfC4vJcBOYbkfkwYK0+gw==",
-      "dev": true,
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.0.0.tgz",
+      "integrity": "sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==",
      "license": "MIT",
      "engines": {
-        "node": ">= 14.18.0"
+        "node": ">= 20.19.0"
      },
      "funding": {
        "type": "individual",
@@ -5553,6 +5552,36 @@
        "typescript": ">=5.0.0"
      }
    },
+    "node_modules/svelte-check/node_modules/chokidar": {
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz",
+      "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "readdirp": "^4.0.1"
+      },
+      "engines": {
+        "node": ">= 14.16.0"
+      },
+      "funding": {
+        "url": "https://paulmillr.com/funding/"
+      }
+    },
+    "node_modules/svelte-check/node_modules/readdirp": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz",
+      "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 14.18.0"
+      },
+      "funding": {
+        "type": "individual",
+        "url": "https://paulmillr.com/funding/"
+      }
+    },
    "node_modules/svelte-i18n": {
      "version": "4.0.1",
      "resolved": "https://registry.npmjs.org/svelte-i18n/-/svelte-i18n-4.0.1.tgz",
--- a/web/package.json
+++ b/web/package.json
@@ -5,6 +5,7 @@
  "scripts": {
    "dev": "vite dev",
    "build": "vite build",
+    "start": "node watcher.js & node build",
    "preview": "vite preview",
    "test": "npm run test:integration && npm run test:unit",
    "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
@@ -58,6 +59,7 @@
    "chart.js": "^4.5.1",
    "chartjs-plugin-crosshair": "^2.0.0",
    "chartjs-plugin-zoom": "^2.1.0",
+    "chokidar": "^5.0.0",
    "crypto-random-string": "^5.0.0",
    "felte": "^1.3.0",
    "heic2any": "^0.0.4",
@@ -85,4 +87,4 @@
      "vite": "$vite"
    }
  }
-}
+}
--- a/web/watcher.js
+++ b/web/watcher.js
@@ -0,0 +1,70 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import chokidar from 'chokidar';
+
+const ORIGIN = "http://localhost:3000";
+const { UPLOAD_FOLDER } = process.env;
+
+if (UPLOAD_FOLDER) {
+  const uploadPath = path.resolve(UPLOAD_FOLDER);
+
+  // Ensure directory exists
+  if (!fs.existsSync(uploadPath)) {
+    try {
+      fs.mkdirSync(uploadPath, { recursive: true });
+    } catch (err) {
+      console.error(`[File Watcher] Failed to create directory ${uploadPath}:`, err.message);
+    }
+  }
+
+  console.log(`[File Watcher] Service active. Watching: ${uploadPath}`);
+
+  const watcher = chokidar.watch(uploadPath, {
+    persistent: true,
+    ignoreInitial: true,
+    awaitWriteFinish: {
+      stabilityThreshold: 2000,
+      pollInterval: 100
+    }
+  });
+
+  watcher.on('add', async (filePath) => {
+    const relative = path.relative(uploadPath, filePath);
+    const [token] = relative.split(path.sep);
+
+    if (!token || token === '.' || path.basename(filePath).startsWith('.')) return;
+
+    try {
+      const fileBuffer = fs.readFileSync(filePath);
+      const fileBlob = new Blob([fileBuffer]);
+      const formData = new FormData();
+      formData.append('file', fileBlob, path.basename(filePath));
+      formData.append('ignoreDuplicates', "true");
+
+      const response = await fetch(`${ORIGIN}/api/v1/trail/upload`, {
+        method: 'PUT',
+        headers: { 'Authorization': `Bearer ${token}` },
+        body: formData
+      });
+
+      if (response.ok) {
+        fs.unlinkSync(filePath);
+        console.log(`[File Watcher] Uploaded and removed: ${path.basename(filePath)}`);
+      } else {
+        const errorText = await response.text();
+        console.error(`[File Watcher] Server rejected ${path.basename(filePath)} (${response.status}): ${errorText}`);
+      }
+    } catch (err) {
+      console.error(`[File Watcher] Upload error:`, err.message);
+    }
+  });
+
+  watcher.on('error', error => console.error(`[File Watcher] Watcher error: ${error}`));
+
+  process.on('SIGTERM', () => {
+    watcher.close();
+  });
+
+} else {
+  console.log('[File Watcher] Disabled: UPLOAD_FOLDER not provided.');
+}