feat: add plugin system (#1034)

* feat: add plugin system * fix db docker build * fix hammerhead readme, add strava subscription news to docs * fixes and sdk improvements * fix: reduce Meilisearch load, debounce federation sync (#1012) * optimize meili trail index * several fixes --------- Co-authored-by: Flomp <Flomp@users.noreply.github.com> * Bump svelte from 5.55.5 to 5.56.0 in /docs (#1032) Bumps [svelte](https://github.com/sveltejs/svelte/tree/HEAD/packages/svelte) from 5.55.5 to 5.56.0. - [Release notes](https://github.com/sveltejs/svelte/releases) - [Changelog](https://github.com/sveltejs/svelte/blob/main/packages/svelte/CHANGELOG.md) - [Commits](https://github.com/sveltejs/svelte/commits/svelte@5.56.0/packages/svelte) --- updated-dependencies: - dependency-name: svelte dependency-version: 5.56.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Flomp <Flomp@users.noreply.github.com> * Release v0.19.2 (#1035) * chore: release v0.19.2 * add changelog --------- Co-authored-by: Flomp <26000991+Flomp@users.noreply.github.com> Co-authored-by: Christian Beutel <> * speed up plugin sync and several small fixes * concepts for security improvements and process stability * improve concept * security concept implemented * remove insecure TLS * worker concept implemented * fixes and cleanup * fixes * docu * mermaid, namings * WASM plugin host improvements, plugin logging * fix db migration * Improve plugin config and category mapping UI * fixes * further fixes * remove manual test sync * fix db migration and strava mapping * type added, UI improvements * fix plugin card toggle clickable area * optimize synch status card layout * plugin type 'trails' instead of 'integration' * session auth validation in UI * fix komoot date and waypoints * improve category mapping * fix send to hammerhead: trail name * plugin setup error handling improved * fix review findings * re-mapping added * rename remote_category --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: Flomp <Flomp@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Flomp <26000991+Flomp@users.noreply.github.com>
2026-06-22 15:00:44 +02:00
parent 2e3d8537b8
commit 485ec53f6d
196 changed files with 20947 additions and 5454 deletions
--- a/db/util/network_test.go
+++ b/db/util/network_test.go
@@ -0,0 +1,68 @@
+package util
+
+import (
+	"bytes"
+	"context"
+	"net"
+	"testing"
+)
+
+func TestFetchPublicURLRejectsUnsafeInputs(t *testing.T) {
+	tests := []string{
+		"ftp://example.com/file.jpg",
+		"http://user:pass@example.com/file.jpg",
+		"http://127.0.0.1/file.jpg",
+		"http://localhost/file.jpg",
+		"http://10.0.0.1/file.jpg",
+		"http://169.254.169.254/latest/meta-data",
+		"http://[::1]/file.jpg",
+		"http://[fc00::1]/file.jpg",
+		"http://example.com:8080/file.jpg",
+	}
+	for _, rawURL := range tests {
+		t.Run(rawURL, func(t *testing.T) {
+			if _, err := FetchPublicURL(context.Background(), rawURL, 1024); err == nil {
+				t.Fatal("expected error")
+			}
+		})
+	}
+}
+
+func TestReadBoundedForPlugin(t *testing.T) {
+	if _, err := ReadBoundedForPlugin(bytes.NewReader([]byte("1234")), 4); err != nil {
+		t.Fatalf("unexpected exact-limit error: %v", err)
+	}
+	if _, err := ReadBoundedForPlugin(bytes.NewReader([]byte("12345")), 4); err == nil {
+		t.Fatal("expected oversized response error")
+	}
+}
+
+func TestConnectorTLSConfigRejectsInsecureMode(t *testing.T) {
+	if _, err := connectorTLSConfig("insecure", nil); err == nil {
+		t.Fatal("expected insecure TLS mode to be rejected")
+	}
+}
+
+func TestConnectorIPAllowed(t *testing.T) {
+	tests := []struct {
+		ip           string
+		allowPrivate bool
+		want         bool
+	}{
+		{ip: "8.8.8.8", want: true},
+		{ip: "10.0.0.1", want: false},
+		{ip: "10.0.0.1", allowPrivate: true, want: true},
+		{ip: "fc00::1", allowPrivate: true, want: true},
+		{ip: "127.0.0.1", allowPrivate: true, want: false},
+		{ip: "169.254.1.1", allowPrivate: true, want: false},
+		{ip: "100.64.0.1", allowPrivate: true, want: false},
+		{ip: "192.0.2.1", allowPrivate: true, want: false},
+	}
+	for _, test := range tests {
+		t.Run(test.ip, func(t *testing.T) {
+			if got := connectorIPAllowed(net.ParseIP(test.ip), test.allowPrivate); got != test.want {
+				t.Fatalf("got %v, want %v", got, test.want)
+			}
+		})
+	}
+}
--- a/db/util/safe_fetch.go
+++ b/db/util/safe_fetch.go
@@ -0,0 +1,226 @@
+package util
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/netip"
+	"net/url"
+	"time"
+
+	"github.com/doyensec/safeurl"
+)
+
+const (
+	DefaultPluginMediaMaxBytes       int64 = 50 << 20
+	DefaultPluginMaxImportMediaItems       = 20
+	DefaultPluginMaxImportMediaBytes int64 = 200 << 20
+)
+
+type SafeFetchResult struct {
+	Body        []byte
+	ContentType string
+	FinalURL    string
+}
+
+type ConnectorHTTPPolicy struct {
+	BaseURL      string
+	AllowPrivate bool
+	TLSMode      string
+	TLSCABundle  []byte
+}
+
+func FetchPublicURL(ctx context.Context, rawURL string, maxBytes int64) (*SafeFetchResult, error) {
+	if maxBytes <= 0 {
+		maxBytes = DefaultPluginMediaMaxBytes
+	}
+	parsed, err := url.Parse(rawURL)
+	if err != nil || parsed.Scheme == "" || parsed.Host == "" {
+		return nil, fmt.Errorf("invalid public URL")
+	}
+	if parsed.User != nil {
+		return nil, fmt.Errorf("public URL must not include credentials")
+	}
+	config := safeurl.GetConfigBuilder().
+		SetTimeout(60*time.Second).
+		SetAllowedSchemes("http", "https").
+		SetAllowedPorts(80, 443).
+		EnableIPv6(true).
+		AllowSendingCredentials(false).
+		SetCheckRedirect(publicMediaRedirectPolicy).
+		Build()
+	client := safeurl.Client(config)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ReadBoundedForPlugin(resp.Body, maxBytes)
+	if err != nil {
+		return nil, err
+	}
+	return &SafeFetchResult{
+		Body:        body,
+		ContentType: resp.Header.Get("Content-Type"),
+		FinalURL:    resp.Request.URL.String(),
+	}, nil
+}
+
+func publicMediaRedirectPolicy(req *http.Request, via []*http.Request) error {
+	if len(via) >= 10 {
+		return fmt.Errorf("too many redirects")
+	}
+	if req.URL.User != nil {
+		return fmt.Errorf("redirect URL must not include credentials")
+	}
+	if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
+		return fmt.Errorf("redirect scheme must be http or https")
+	}
+	if len(via) > 0 && via[len(via)-1].URL.Scheme == "https" && req.URL.Scheme == "http" {
+		return fmt.Errorf("redirect downgrades https to http")
+	}
+	return nil
+}
+
+func ConnectorHTTPClient(policy ConnectorHTTPPolicy, checkRedirect func(req *http.Request, via []*http.Request) error) (*http.Client, error) {
+	base, err := url.Parse(policy.BaseURL)
+	if err != nil || base.Scheme == "" || base.Host == "" {
+		return nil, fmt.Errorf("invalid connector baseURL")
+	}
+	tlsConfig, err := connectorTLSConfig(policy.TLSMode, policy.TLSCABundle)
+	if err != nil {
+		return nil, err
+	}
+	dialer := &net.Dialer{Timeout: 30 * time.Second}
+	transport := &http.Transport{
+		TLSClientConfig: tlsConfig,
+		DialContext: func(ctx context.Context, network string, addr string) (net.Conn, error) {
+			host, port, err := net.SplitHostPort(addr)
+			if err != nil {
+				return nil, err
+			}
+			ips, err := net.DefaultResolver.LookupIP(ctx, "ip", host)
+			if err != nil || len(ips) == 0 {
+				return nil, fmt.Errorf("failed to resolve connector host: %w", err)
+			}
+			var selected net.IP
+			for _, ip := range ips {
+				if connectorIPAllowed(ip, policy.AllowPrivate) {
+					selected = ip
+					break
+				}
+			}
+			if selected == nil {
+				return nil, fmt.Errorf("connector host resolved outside allowed IP policy")
+			}
+			return dialer.DialContext(ctx, network, net.JoinHostPort(selected.String(), port))
+		},
+	}
+	return &http.Client{
+		Timeout:       60 * time.Second,
+		Transport:     transport,
+		CheckRedirect: checkRedirect,
+	}, nil
+}
+
+func connectorTLSConfig(mode string, caBundle []byte) (*tls.Config, error) {
+	switch mode {
+	case "", "system":
+		return nil, nil
+	case "customCA":
+		roots, err := x509.SystemCertPool()
+		if err != nil || roots == nil {
+			roots = x509.NewCertPool()
+		}
+		if len(caBundle) == 0 || !roots.AppendCertsFromPEM(caBundle) {
+			return nil, fmt.Errorf("connector customCA bundle is invalid")
+		}
+		return &tls.Config{RootCAs: roots}, nil
+	default:
+		return nil, fmt.Errorf("unsupported connector TLS mode %q", mode)
+	}
+}
+
+func connectorIPAllowed(ip net.IP, allowPrivate bool) bool {
+	addr, ok := netip.AddrFromSlice(ip)
+	if !ok {
+		return false
+	}
+	if addr.Is4In6() {
+		addr = addr.Unmap()
+	}
+	if addr.IsLoopback() || addr.IsLinkLocalUnicast() || addr.IsLinkLocalMulticast() ||
+		addr.IsMulticast() || addr.IsUnspecified() {
+		return false
+	}
+	if isSpecialPurposeIP(addr) {
+		return false
+	}
+	if addr.IsPrivate() {
+		return allowPrivate
+	}
+	return true
+}
+
+func isSpecialPurposeIP(addr netip.Addr) bool {
+	for _, prefix := range specialPurposePrefixes {
+		if prefix.Contains(addr) {
+			return true
+		}
+	}
+	return false
+}
+
+var specialPurposePrefixes = mustPrefixes(
+	"0.0.0.0/8",
+	"100.64.0.0/10",
+	"127.0.0.0/8",
+	"169.254.0.0/16",
+	"192.0.0.0/24",
+	"192.0.2.0/24",
+	"198.18.0.0/15",
+	"198.51.100.0/24",
+	"203.0.113.0/24",
+	"224.0.0.0/4",
+	"240.0.0.0/4",
+	"::/128",
+	"::1/128",
+	"64:ff9b::/96",
+	"100::/64",
+	"2001:db8::/32",
+	"fe80::/10",
+	"ff00::/8",
+)
+
+func mustPrefixes(values ...string) []netip.Prefix {
+	prefixes := make([]netip.Prefix, 0, len(values))
+	for _, value := range values {
+		prefix, err := netip.ParsePrefix(value)
+		if err != nil {
+			panic(err)
+		}
+		prefixes = append(prefixes, prefix)
+	}
+	return prefixes
+}
+
+func ReadBoundedForPlugin(reader io.Reader, maxBytes int64) ([]byte, error) {
+	body, err := io.ReadAll(io.LimitReader(reader, maxBytes+1))
+	if err != nil {
+		return nil, err
+	}
+	if int64(len(body)) > maxBytes {
+		return nil, fmt.Errorf("response exceeds maximum size")
+	}
+	return body, nil
+}
--- a/db/util/trail_access.go
+++ b/db/util/trail_access.go
@@ -0,0 +1,45 @@
+package util
+
+import (
+	"github.com/pocketbase/dbx"
+	"github.com/pocketbase/pocketbase/core"
+)
+
+// TrailViewableByUser mirrors the trails view/read rule for custom backend
+// routes that load a trail server-side and therefore bypass PocketBase's normal
+// collection API permission checks.
+func TrailViewableByUser(app core.App, trail *core.Record, userID string, shareToken string) bool {
+	if trail == nil || userID == "" {
+		return false
+	}
+	if trail.GetBool("public") {
+		return true
+	}
+
+	actor, err := app.FindFirstRecordByData("activitypub_actors", "user", userID)
+	if err != nil {
+		return false
+	}
+	if trail.GetString("author") == actor.Id {
+		return true
+	}
+
+	share, err := app.FindFirstRecordByFilter(
+		"trail_share",
+		"trail={:trail} && actor={:actor}",
+		dbx.Params{"trail": trail.Id, "actor": actor.Id},
+	)
+	if err == nil && share != nil {
+		return true
+	}
+
+	if shareToken == "" {
+		return false
+	}
+	linkShare, err := app.FindFirstRecordByFilter(
+		"trail_link_share",
+		"trail={:trail} && token={:token}",
+		dbx.Params{"trail": trail.Id, "token": shareToken},
+	)
+	return err == nil && linkShare != nil
+}
--- a/db/util/trail_external_reference.go
+++ b/db/util/trail_external_reference.go
@@ -1,30 +1,38 @@
 package util

 import (
+	"database/sql"
+	"errors"
 	"fmt"
+	"strings"
+	"time"

 	"github.com/pocketbase/dbx"
 	"github.com/pocketbase/pocketbase/core"
 )

-func FindTrailByExternalReference(app core.App, provider string, externalID string) (*core.Record, error) {
-	if provider == "" || externalID == "" {
+func FindTrailByExternalReferenceForUser(app core.App, userID string, provider string, externalID string) (*core.Record, error) {
+	if userID == "" || provider == "" || externalID == "" {
 		return nil, nil
 	}

 	refs, err := app.FindRecordsByFilter(
 		"trail_external_reference",
-		"provider={:provider} && external_id={:external_id}",
+		"user={:user} && provider={:provider} && external_id={:external_id}",
 		"+created",
 		1,
 		0,
 		dbx.Params{
+			"user":        userID,
 			"provider":    provider,
 			"external_id": externalID,
 		},
 	)
 	if err != nil || len(refs) == 0 {
-		return nil, err
+		if err != nil {
+			return nil, err
+		}
+		return nil, nil
 	}

 	trailID := refs[0].GetString("trail")
@@ -32,21 +40,105 @@ func FindTrailByExternalReference(app core.App, provider string, externalID stri
 		return nil, nil
 	}

-	return app.FindRecordById("trails", trailID)
+	trail, err := app.FindRecordById("trails", trailID)
+	if err == nil {
+		return trail, nil
+	}
+	if !errors.Is(err, sql.ErrNoRows) {
+		return nil, err
+	}
+
+	if deleteErr := app.Delete(refs[0]); deleteErr != nil {
+		return nil, fmt.Errorf("delete orphaned trail external reference: %w", deleteErr)
+	}
+	app.Logger().Warn("deleted orphaned trail external reference", "provider", provider, "external_id", externalID, "trail", trailID)
+	return nil, nil
 }

-func EnsureTrailExternalReference(app core.App, trailID string, provider string, externalID string) error {
+func FindExistingExternalReferenceIDsForUser(app core.App, userID string, provider string, externalIDs []string) (map[string]bool, error) {
+	existingIDs := map[string]bool{}
+	if userID == "" || provider == "" || len(externalIDs) == 0 {
+		return existingIDs, nil
+	}
+
+	params := dbx.Params{
+		"user":     userID,
+		"provider": provider,
+	}
+	seen := map[string]bool{}
+	idFilters := make([]string, 0, len(externalIDs))
+	for _, externalID := range externalIDs {
+		if externalID == "" || seen[externalID] {
+			continue
+		}
+		seen[externalID] = true
+		paramName := fmt.Sprintf("external_id_%d", len(idFilters))
+		params[paramName] = externalID
+		idFilters = append(idFilters, "external_id={:"+paramName+"}")
+	}
+	if len(idFilters) == 0 {
+		return existingIDs, nil
+	}
+
+	filter := "user={:user} && provider={:provider} && (" + strings.Join(idFilters, " || ") + ")"
+	refs, err := app.FindRecordsByFilter("trail_external_reference", filter, "", len(idFilters), 0, params)
+	if err != nil || len(refs) == 0 {
+		return existingIDs, err
+	}
+
+	trailIDs := make([]string, 0, len(refs))
+	for _, ref := range refs {
+		if trailID := ref.GetString("trail"); trailID != "" {
+			trailIDs = append(trailIDs, trailID)
+		}
+	}
+	var trails []*core.Record
+	if len(trailIDs) > 0 {
+		trails, err = app.FindRecordsByIds("trails", trailIDs)
+		if err != nil {
+			return nil, err
+		}
+	}
+	trailsByID := make(map[string]bool, len(trails))
+	for _, trail := range trails {
+		trailsByID[trail.Id] = true
+	}
+
+	for _, ref := range refs {
+		trailID := ref.GetString("trail")
+		if trailID != "" && trailsByID[trailID] {
+			existingIDs[ref.GetString("external_id")] = true
+			continue
+		}
+		if deleteErr := app.Delete(ref); deleteErr != nil {
+			return nil, fmt.Errorf("delete orphaned trail external reference: %w", deleteErr)
+		}
+		app.Logger().Warn("deleted orphaned trail external reference", "provider", provider, "external_id", ref.GetString("external_id"), "trail", trailID)
+	}
+	return existingIDs, nil
+}
+
+func EnsureTrailExternalReference(app core.App, trailID string, provider string, externalID string, pluginID string, providerCategory string) error {
 	if trailID == "" || provider == "" || externalID == "" {
 		return nil
 	}
+	userID, err := externalReferenceUserID(app, trailID)
+	if err != nil {
+		return err
+	}
+	if userID == "" {
+		app.Logger().Warn("skipping trail external reference without local user", "provider", provider, "external_id", externalID, "trail", trailID)
+		return nil
+	}

 	refs, err := app.FindRecordsByFilter(
 		"trail_external_reference",
-		"provider={:provider} && external_id={:external_id}",
+		"user={:user} && provider={:provider} && external_id={:external_id}",
 		"",
 		1,
 		0,
 		dbx.Params{
+			"user":        userID,
 			"provider":    provider,
 			"external_id": externalID,
 		},
@@ -56,6 +148,19 @@ func EnsureTrailExternalReference(app core.App, trailID string, provider string,
 	}
 	if len(refs) > 0 {
 		if refs[0].GetString("trail") == trailID {
+			changed := false
+			if pluginID != "" && refs[0].GetString("plugin_id") == "" {
+				refs[0].Set("plugin_id", pluginID)
+				changed = true
+			}
+			if refs[0].GetDateTime("provider_category_checked_at").IsZero() {
+				refs[0].Set("provider_category", providerCategory)
+				refs[0].Set("provider_category_checked_at", time.Now())
+				changed = true
+			}
+			if changed {
+				return app.Save(refs[0])
+			}
 			return nil
 		}
 		return fmt.Errorf("trail external reference already exists for another trail")
@@ -68,14 +173,30 @@ func EnsureTrailExternalReference(app core.App, trailID string, provider string,

 	record := core.NewRecord(collection)
 	record.Load(map[string]any{
-		"trail":       trailID,
-		"provider":    provider,
-		"external_id": externalID,
+		"trail":                        trailID,
+		"user":                         userID,
+		"provider":                     provider,
+		"external_id":                  externalID,
+		"plugin_id":                    pluginID,
+		"provider_category":            providerCategory,
+		"provider_category_checked_at": time.Now(),
 	})

 	return app.Save(record)
 }

+func externalReferenceUserID(app core.App, trailID string) (string, error) {
+	trail, err := app.FindRecordById("trails", trailID)
+	if err != nil {
+		return "", err
+	}
+	actor, err := app.FindRecordById("activitypub_actors", trail.GetString("author"))
+	if err != nil {
+		return "", err
+	}
+	return actor.GetString("user"), nil
+}
+
 func ReassignTrailExternalReferences(app core.App, sourceTrailID string, targetTrailID string) error {
 	if sourceTrailID == "" || targetTrailID == "" || sourceTrailID == targetTrailID {
 		return nil