diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 68458977..a2d82321 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -18,11 +18,11 @@ jobs: steps: # 1. Checkout the repository - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v5 # 2. Setup node & npm - name: Setup Node.js - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: '22' @@ -53,12 +53,12 @@ jobs: steps: # 1. Checkout the repository - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v5 with: ref: ${{ github.ref }} - name: Setup Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: '1.22' @@ -106,7 +106,7 @@ jobs: steps: # 1. Checkout the repository - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v5 with: ref: ${{ github.ref }} # 2. Extract release notes from CHANGELOG.md diff --git a/.gitignore b/.gitignore index 61487c08..5c5302b5 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,6 @@ search/dumps run.sh build*.sh -start.* +start*.* data*/ diff --git a/CHANGELOG.md b/CHANGELOG.md index eb2c139c..c605d0ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +##v0.18.2 +## Features +- Adds `dedup` command to pocketbase. This command allows an admin to quickly identify duplicate trails and delete them. Use the `--dry-run` flag to only log duplicate trails without deleting them. To execute the command run `docker exec -it wanderer-db ./pocketbase dedup --dry-run`. +- Adds option to only sync strava activities after a certain date +- Singificant performance improvements for instances with larger userbases +- Greatly improved initial indexing speed when starting wanderer +## Bug fixes +- Fixes permission issues for public trails +- Fixes bug that caused trails to be duplicated multiple times (to clean up see the `dedup` command above) +- Fixes link to "New Trail" from empty profiles +- Fixes link when opening a trail from the map searchbar +- Sorting by difficulty no longer sorts by difficulty alphabetically +- Fixes strava integration stopping after only one page + # v0.18.1 ## Bug fixes - Fixes permission issues that prevented federation from working properly diff --git a/db/commands/dedup.go b/db/commands/dedup.go new file mode 100644 index 00000000..ec3fb524 --- /dev/null +++ b/db/commands/dedup.go @@ -0,0 +1,121 @@ +package commands + +import ( + "crypto/sha1" + "fmt" + "log" + "sort" + + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/core" + "github.com/spf13/cobra" +) + +func Dedup(app *pocketbase.PocketBase) *cobra.Command { + var dryRun bool + + cmd := &cobra.Command{ + Use: "dedup", + Short: "Deduplicate trails by all matching fields", + Run: func(cmd *cobra.Command, args []string) { + records, err := app.FindAllRecords("trails") + if err != nil { + log.Fatalf("failed to fetch trails: %v", err) + } + + // group by composite key + trailsByKey := make(map[string][]*core.Record) + for _, r := range records { + key := makeKey(r) + trailsByKey[key] = append(trailsByKey[key], r) + } + + var duplicates []*core.Record + for _, recs := range trailsByKey { + if len(recs) <= 1 { + continue + } + + // sort by created date ascending + sort.Slice(recs, func(i, j int) bool { + return recs[i].GetDateTime("created").Time().Before(recs[j].GetDateTime("created").Time()) + }) + + original := recs[0] + dupes := recs[1:] + + // print header row for original + // print original as header + fmt.Printf("\nOriginal: id=%s, name=%s, distance=%.2f, elevation_gain=%.2f, elevation_loss=%.2f, lat=%.5f, lon=%.5f, duration=%.2f, location=%s, category=%s, author=%s, created=%s\n", + original.Id, + original.GetString("name"), + original.GetFloat("distance"), + original.GetFloat("elevation_gain"), + original.GetFloat("elevation_loss"), + original.GetFloat("lat"), + original.GetFloat("lon"), + original.GetFloat("duration"), + original.GetString("location"), + original.GetString("category"), + original.GetString("author"), + original.GetDateTime("created"), + ) + + // print duplicates indented + for _, d := range dupes { + fmt.Printf(" Duplicate: id=%s, name=%s, distance=%.2f, elevation_gain=%.2f, elevation_loss=%.2f, lat=%.5f, lon=%.5f, duration=%.2f, location=%s, category=%s, author=%s, created=%s\n", + d.Id, + d.GetString("name"), + d.GetFloat("distance"), + d.GetFloat("elevation_gain"), + d.GetFloat("elevation_loss"), + d.GetFloat("lat"), + d.GetFloat("lon"), + d.GetFloat("duration"), + d.GetString("location"), + d.GetString("category"), + d.GetString("author"), + d.GetDateTime("created"), + ) + duplicates = append(duplicates, d) + } + } + + if dryRun { + fmt.Printf("\n[Dry Run] Found %d duplicates (no deletions performed)\n", len(duplicates)) + return + } + + // delete duplicates + for _, d := range duplicates { + if err := app.Delete(d); err != nil { + fmt.Printf("Failed to delete duplicate %s: %v\n", d.Id, err) + } else { + fmt.Printf("Deleted duplicate %s\n", d.Id) + } + } + }, + } + + cmd.Flags().BoolVar(&dryRun, "dry-run", false, "Show duplicates without deleting them") + + return cmd +} + +// makeKey creates a composite key string for duplicate detection +func makeKey(r *core.Record) string { + data := fmt.Sprintf("%s|%f|%f|%f|%f|%f|%f|%s|%s|%s", + r.GetString("name"), + r.GetFloat("distance"), + r.GetFloat("elevation_gain"), + r.GetFloat("elevation_loss"), + r.GetFloat("lat"), + r.GetFloat("lon"), + r.GetFloat("duration"), + r.GetString("location"), + r.GetString("category"), + r.GetString("author"), + ) + h := sha1.Sum([]byte(data)) + return fmt.Sprintf("%x", h) +} diff --git a/db/integrations/komoot/komoot.go b/db/integrations/komoot/komoot.go index a2041d5a..51b62ee8 100644 --- a/db/integrations/komoot/komoot.go +++ b/db/integrations/komoot/komoot.go @@ -205,27 +205,27 @@ func syncTrailWithTours(app core.App, k *KomootApi, i KomootIntegration, user st app.Logger().Warn(fmt.Sprintf("Unable to generate GPX for tour '%s': %v", tour.Name, err)) continue } - wpIds, err := createWaypointsFromTour(app, detailedTour, user) - if err != nil { - app.Logger().Warn(fmt.Sprintf("Unable to create waypoints for tour '%s': %v", tour.Name, err)) - continue - } - err = createTrailFromTour(app, k, detailedTour, gpx, actor, wpIds) + trailid, err := createTrailFromTour(app, k, detailedTour, gpx, actor) if err != nil { app.Logger().Warn(fmt.Sprintf("Unable to create trail for tour '%s': %v", tour.Name, err)) continue } + err = createWaypointsFromTour(app, detailedTour, user, trailid) + if err != nil { + app.Logger().Warn(fmt.Sprintf("Unable to create waypoints for tour '%s': %v", tour.Name, err)) + continue + } } return hasNewTours, nil } -func createTrailFromTour(app core.App, k *KomootApi, detailedTour *DetailedKomootTour, gpx *filesystem.File, actor string, wpIds []string) error { +func createTrailFromTour(app core.App, k *KomootApi, detailedTour *DetailedKomootTour, gpx *filesystem.File, actor string) (string, error) { trailid := security.RandomStringWithAlphabet(core.DefaultIdLength, core.DefaultIdAlphabet) collection, err := app.FindCollectionByNameOrId("trails") if err != nil { - return err + return "", err } record := core.NewRecord(collection) @@ -251,12 +251,12 @@ func createTrailFromTour(app core.App, k *KomootApi, detailedTour *DetailedKomoo if len(detailedTour.Embedded.CoverImages.Embedded.Items) > 0 { photos, err = fetchRoutePhotos(k, detailedTour) if err != nil { - return err + return "", err } } else { photo, err := fetchPhoto(detailedTour.MapImage.Src, "", "") if err != nil { - return err + return "", err } photos = append(photos, photo) } @@ -281,7 +281,6 @@ func createTrailFromTour(app core.App, k *KomootApi, detailedTour *DetailedKomoo "lon": detailedTour.StartPoint.Lng, "difficulty": diffculty, "category": categoryId, - "waypoints": wpIds, "author": actor, }) @@ -293,13 +292,13 @@ func createTrailFromTour(app core.App, k *KomootApi, detailedTour *DetailedKomoo } if err := app.Save(record); err != nil { - return err + return "", err } if detailedTour.Type == "tour_recorded" { collection, err := app.FindCollectionByNameOrId("summit_logs") if err != nil { - return err + return "", err } summitLogRecord := core.NewRecord(collection) @@ -313,25 +312,23 @@ func createTrailFromTour(app core.App, k *KomootApi, detailedTour *DetailedKomoo "trail": trailid, }) if err := app.Save(summitLogRecord); err != nil { - return err + return "", err } } - return nil + return trailid, nil } -func createWaypointsFromTour(app core.App, tour *DetailedKomootTour, user string) ([]string, error) { +func createWaypointsFromTour(app core.App, tour *DetailedKomootTour, user string, trailid string) error { collection, err := app.FindCollectionByNameOrId("waypoints") if err != nil { - return nil, err + return err } - wpIds := make([]string, len(tour.Embedded.Timeline.Embedded.Items)) - - for i, wp := range tour.Embedded.Timeline.Embedded.Items { + for _, wp := range tour.Embedded.Timeline.Embedded.Items { photos, err := fetchWaypointPhotos(wp) if err != nil { - return nil, err + return err } record := core.NewRecord(collection) @@ -358,6 +355,7 @@ func createWaypointsFromTour(app core.App, tour *DetailedKomootTour, user string "icon": "circle", "author": user, "distance_from_start": 0, + "trail": trailid, }) if photos != nil { @@ -365,13 +363,11 @@ func createWaypointsFromTour(app core.App, tour *DetailedKomootTour, user string } if err := app.Save(record); err != nil { - return nil, err + return err } - - wpIds[i] = record.Id } - return wpIds, nil + return nil } func fetchRoutePhotos(k *KomootApi, tour *DetailedKomootTour) ([]*filesystem.File, error) { diff --git a/db/integrations/strava/models.go b/db/integrations/strava/models.go index 6b8fead8..8fa4f0fd 100644 --- a/db/integrations/strava/models.go +++ b/db/integrations/strava/models.go @@ -29,6 +29,7 @@ type StravaIntegration struct { AccessToken string `json:"accessToken,omitempty"` RefreshToken string `json:"refreshToken,omitempty"` ExpiresAt int64 `json:"expiresAt,omitempty"` + After string `json:"after,omitempty"` } type StravaRoute struct { Athlete Athlete `json:"athlete"` diff --git a/db/integrations/strava/strava.go b/db/integrations/strava/strava.go index e291ea9d..bf347381 100644 --- a/db/integrations/strava/strava.go +++ b/db/integrations/strava/strava.go @@ -89,22 +89,12 @@ func SyncStrava(app core.App) error { stravaIntegration.ExpiresAt = r.ExpiresAt } - b, err := json.Marshal(stravaIntegration) - if err != nil { - return err - } - i.Set("strava", string(b)) - err = app.Save(i) - if err != nil { - return err - } - if stravaIntegration.Routes { page := 1 - hasNewRoutes := true - for hasNewRoutes { - + hasMore := true + for hasMore { routes, err := fetchStravaRoutes(r.AccessToken, page) + hasMore = len(routes) > 0 page += 1 if err != nil { warning := fmt.Sprintf("error fetching routes from strava: %v\n", err) @@ -112,7 +102,7 @@ func SyncStrava(app core.App) error { app.Logger().Warn(warning) break } - hasNewRoutes, err = syncTrailsWithRoutes(app, r.AccessToken, userId, actorId, routes) + err = syncTrailsWithRoutes(app, r.AccessToken, userId, actorId, routes) if err != nil { warning := fmt.Sprintf("error syncing strava routes with trails: %v\n", err) fmt.Print(warning) @@ -123,9 +113,20 @@ func SyncStrava(app core.App) error { } if stravaIntegration.Activities { page := 1 - hasNewActivities := true - for hasNewActivities { - activities, err := fetchStravaActivities(r.AccessToken, page) + hasMore := true + for hasMore { + var after int64 = 0 + if stravaIntegration.After != "" { + t, err := time.Parse("2006-01-02", stravaIntegration.After) + if err != nil { + return err + } + t = t.UTC() + + after = t.Unix() + } + activities, err := fetchStravaActivities(r.AccessToken, page, after) + hasMore = len(activities) > 0 page += 1 if err != nil { warning := fmt.Sprintf("error fetching activities from strava: %v", err) @@ -133,7 +134,8 @@ func SyncStrava(app core.App) error { app.Logger().Warn(warning) break } - hasNewActivities, err = syncTrailsWithActivities(app, r.AccessToken, userId, actorId, activities) + err = syncTrailsWithActivities(app, r.AccessToken, actorId, activities) + if err != nil { warning := fmt.Sprintf("error syncing strava activities with trails: %v", err) fmt.Print(warning) @@ -141,6 +143,17 @@ func SyncStrava(app core.App) error { continue } } + + } + + b, err := json.Marshal(stravaIntegration) + if err != nil { + return err + } + i.Set("strava", string(b)) + err = app.Save(i) + if err != nil { + return err } } @@ -208,8 +221,8 @@ func fetchStravaRoutes(accessToken string, page int) ([]StravaRoute, error) { return routes, nil } -func fetchStravaActivities(accessToken string, page int) ([]StravaActivity, error) { - stravaRoutesURL := fmt.Sprintf("https://www.strava.com/api/v3/athlete/activities?page=%d", page) +func fetchStravaActivities(accessToken string, page int, after int64) ([]StravaActivity, error) { + stravaRoutesURL := fmt.Sprintf("https://www.strava.com/api/v3/athlete/activities?page=%d&after=%d", page, after) req, err := http.NewRequest("GET", stravaRoutesURL, nil) if err != nil { return nil, err @@ -235,36 +248,33 @@ func fetchStravaActivities(accessToken string, page int) ([]StravaActivity, erro return activities, nil } -func syncTrailsWithRoutes(app core.App, accessToken string, user string, actor string, routes []StravaRoute) (bool, error) { - hasNewRoutes := false +func syncTrailsWithRoutes(app core.App, accessToken string, user string, actor string, routes []StravaRoute) error { for _, route := range routes { trails, err := app.FindRecordsByFilter("trails", "external_id = {:id}", "", 1, 0, dbx.Params{"id": route.IDStr}) if err != nil { - return hasNewRoutes, err + return err } if len(trails) != 0 { continue } - hasNewRoutes = true gpx, err := fetchRouteGPX(route, accessToken) if err != nil { app.Logger().Warn(fmt.Sprintf("Unable to fetch GPX for route '%s': %v", route.Name, err)) continue } - wpIds, err := createWaypointsFromRoute(app, route, user) - if err != nil { - app.Logger().Warn(fmt.Sprintf("Unable to create waypoints for route '%s': %v", route.Name, err)) - continue - } - err = createTrailFromRoute(app, route, gpx, actor, wpIds) + trailid, err := createTrailFromRoute(app, route, gpx, actor) if err != nil { app.Logger().Warn(fmt.Sprintf("Unable to create trail for route '%s': %v", route.Name, err)) continue } - + err = createWaypointsFromRoute(app, route, user, trailid) + if err != nil { + app.Logger().Warn(fmt.Sprintf("Unable to create waypoints for route '%s': %v", route.Name, err)) + continue + } } - return hasNewRoutes, nil + return nil } func fetchRouteGPX(route StravaRoute, accessToken string) (*filesystem.File, error) { @@ -305,10 +315,12 @@ func fetchRouteGPX(route StravaRoute, accessToken string) (*filesystem.File, err return gpxFile, nil } -func createTrailFromRoute(app core.App, route StravaRoute, gpx *filesystem.File, actor string, wpIds []string) error { +func createTrailFromRoute(app core.App, route StravaRoute, gpx *filesystem.File, actor string) (string, error) { + trailid := security.RandomStringWithAlphabet(core.DefaultIdLength, core.DefaultIdAlphabet) + collection, err := app.FindCollectionByNameOrId("trails") if err != nil { - return err + return "", err } record := core.NewRecord(collection) @@ -337,6 +349,7 @@ func createTrailFromRoute(app core.App, route StravaRoute, gpx *filesystem.File, } record.Load(map[string]any{ + "id": trailid, "name": route.Name, "description": route.Description, "public": !route.Private, @@ -348,7 +361,6 @@ func createTrailFromRoute(app core.App, route StravaRoute, gpx *filesystem.File, "external_id": route.IDStr, "lat": lat, "lon": lon, - "waypoints": wpIds, "difficulty": "easy", "category": category, "author": actor, @@ -359,20 +371,18 @@ func createTrailFromRoute(app core.App, route StravaRoute, gpx *filesystem.File, } if err := app.Save(record); err != nil { - return err + return "", err } - return nil + return trailid, err } -func createWaypointsFromRoute(app core.App, route StravaRoute, user string) ([]string, error) { +func createWaypointsFromRoute(app core.App, route StravaRoute, user string, trailid string) error { collection, err := app.FindCollectionByNameOrId("waypoints") if err != nil { - return nil, err + return err } - wpIds := make([]string, len(route.Waypoints)) - for i, wp := range route.Waypoints { record := core.NewRecord(collection) @@ -383,26 +393,26 @@ func createWaypointsFromRoute(app core.App, route StravaRoute, user string) ([]s record.Set("icon", "circle") record.Set("author", user) record.Set("distance_from_start", wp.DistanceIntoRoute) + record.Set("trail", trailid) - app.Save(record) + if err := app.Save(record); err != nil { + return err + } - wpIds[i] = record.Id } - return wpIds, nil + return nil } -func syncTrailsWithActivities(app core.App, accessToken string, user string, actor string, activities []StravaActivity) (bool, error) { - hasNewActivites := false +func syncTrailsWithActivities(app core.App, accessToken string, actor string, activities []StravaActivity) error { for _, activity := range activities { trails, err := app.FindRecordsByFilter("trails", "external_id = {:id}", "", 1, 0, dbx.Params{"id": strconv.Itoa(int(activity.ID))}) if err != nil { - return hasNewActivites, err + return err } if len(trails) != 0 { continue } - hasNewActivites = true detailedActivity, err := fetchDetailedActivity(activity, accessToken) if err != nil { app.Logger().Warn(fmt.Sprintf("Unable to fetch detailed activity '%s': %v", activity.Name, err)) @@ -418,10 +428,9 @@ func syncTrailsWithActivities(app core.App, accessToken string, user string, act app.Logger().Warn(fmt.Sprintf("Unable to create trail from activity '%s': %v", activity.Name, err)) continue } - } - return hasNewActivites, nil + return nil } func fetchDetailedActivity(activity StravaActivity, accessToken string) (*DetailedStravaActivity, error) { diff --git a/db/main.go b/db/main.go index 2245b938..67b5957f 100644 --- a/db/main.go +++ b/db/main.go @@ -21,6 +21,7 @@ import ( "github.com/pocketbase/pocketbase/tools/security" "github.com/spf13/cast" + "pocketbase/commands" "pocketbase/federation" "pocketbase/integrations/komoot" "pocketbase/integrations/strava" @@ -73,6 +74,8 @@ func main() { registerMigrations(app) setupEventHandlers(app, client) + setupCommands(app) + if err := app.Start(); err != nil { log.Fatal(err) } @@ -140,6 +143,10 @@ func setupEventHandlers(app *pocketbase.PocketBase, client meilisearch.ServiceMa app.OnBootstrap().BindFunc(onBootstrapHandler()) } +func setupCommands(app *pocketbase.PocketBase) { + app.RootCmd.AddCommand(commands.Dedup(app)) +} + func sanitizeHTML() func(e *core.RecordRequestEvent) error { return func(e *core.RecordRequestEvent) error { fieldsToSanitize := map[string][]string{ @@ -233,7 +240,7 @@ func createTrailHandler(client meilisearch.ServiceManager) func(e *core.RecordEv if err != nil { return err } - if err := util.IndexTrail(e.App, record, author, client); err != nil { + if err := util.IndexTrails(e.App, []*core.Record{record}, client); err != nil { return err } if !author.GetBool("isLocal") { @@ -340,12 +347,7 @@ func createSummitLogHandler(client meilisearch.ServiceManager) func(e *core.Reco return err } - trailAuthor, err := e.App.FindFirstRecordByData("activitypub_actors", "id", trail.GetString("author")) - if err != nil { - return err - } - - if err := util.IndexTrail(e.App, trail, trailAuthor, client); err != nil { + if err := util.IndexTrails(e.App, []*core.Record{trail}, client); err != nil { return err } @@ -391,12 +393,7 @@ func deleteSummitLogHandler(client meilisearch.ServiceManager) func(e *core.Reco return err } - trailAuthor, err := e.App.FindFirstRecordByData("activitypub_actors", "id", trail.GetString("author")) - if err != nil { - return err - } - - if err := util.IndexTrail(e.App, trail, trailAuthor, client); err != nil { + if err := util.IndexTrails(e.App, []*core.Record{trail}, client); err != nil { return err } @@ -616,7 +613,7 @@ func createListHandler(client meilisearch.ServiceManager) func(e *core.RecordEve return err } - if err := util.IndexList(e.App, record, author, client); err != nil { + if err := util.IndexLists(e.App, []*core.Record{record}, client); err != nil { return err } @@ -1308,94 +1305,62 @@ func bootstrapCategories(app core.App) error { } func bootstrapMeilisearchDocuments(app core.App, client meilisearch.ServiceManager) error { - query := app.RecordQuery("trails") - trails := []*core.Record{} + // --- Trails --- + const pageSize int64 = 100 + var page int64 = 0 - if err := query.All(&trails); err != nil { + // Clear index before re-indexing + if _, err := client.Index("trails").DeleteAllDocuments(); err != nil { return err } - _, err := client.Index("trails").DeleteAllDocuments() - if err != nil { - return err - } - for _, trail := range trails { - author, err := app.FindRecordById("activitypub_actors", trail.GetString(("author"))) + for { + trails := []*core.Record{} + err := app.RecordQuery("trails"). + Limit(pageSize). + Offset(page * pageSize). + All(&trails) if err != nil { return err } - if err := util.IndexTrail(app, trail, author, client); err != nil { - app.Logger().Warn(fmt.Sprintf("Unable to index trail '%s': %v", trail.GetString("name"), err)) + if len(trails) == 0 { + break + } + + if err := util.IndexTrails(app, trails, client); err != nil { + app.Logger().Warn(fmt.Sprintf("Unable to index trails page %d: %v", page, err)) continue } - shares, err := app.FindAllRecords("trail_share", - dbx.NewExp("trail = {:trailId}", dbx.Params{"trailId": trail.Id}), - ) - if err != nil { - return err - } - actorIds := make([]string, len(shares)) - for i, r := range shares { - actorIds[i] = r.GetString("actor") - } - err = util.UpdateTrailShares(trail.Id, actorIds, client) - if err != nil { - app.Logger().Warn(fmt.Sprintf("Unable to update trail shares '%s': %v", trail.GetString("name"), err)) - continue - } - likes, err := app.FindAllRecords("trail_like", - dbx.NewExp("trail = {:trailId}", dbx.Params{"trailId": trail.Id}), - ) - if err != nil { - return err - } - actorIds = make([]string, len(likes)) - for i, r := range likes { - actorIds[i] = r.GetString("actor") - } - err = util.UpdateTrailLikes(trail.Id, actorIds, client) - if err != nil { - app.Logger().Warn(fmt.Sprintf("Unable to update trail likes '%s': %v", trail.GetString("name"), err)) - continue - } + page++ } - lists, err := app.FindAllRecords("lists") - if err != nil { - return err - } - _, err = client.Index("lists").DeleteAllDocuments() - if err != nil { + // --- Lists --- + if _, err := client.Index("lists").DeleteAllDocuments(); err != nil { return err } - for _, list := range lists { - author, err := app.FindRecordById("activitypub_actors", list.GetString(("author"))) + page = 0 + for { + lists := []*core.Record{} + err := app.RecordQuery("lists"). + Limit(pageSize). + Offset(page * pageSize). + All(&lists) if err != nil { return err } - if err := util.IndexList(app, list, author, client); err != nil { - app.Logger().Warn(fmt.Sprintf("Unable to index list '%s': %v", list.GetString("name"), err)) + if len(lists) == 0 { + break + } + + if err := util.IndexLists(app, lists, client); err != nil { + app.Logger().Warn(fmt.Sprintf("Unable to index list page %d: %v", page, err)) continue } - shares, err := app.FindAllRecords("list_share", - dbx.NewExp("list = {:listId}", dbx.Params{"listId": list.Id}), - ) - if err != nil { - return err - } - actorIds := make([]string, len(shares)) - for i, r := range shares { - actorIds[i] = r.GetString("actor") - } - err = util.UpdateListShares(list.Id, actorIds, client) - - if err != nil { - app.Logger().Warn(fmt.Sprintf("Unable to update list shares '%s': %v", list.GetString("name"), err)) - continue - } + page++ } + return nil } diff --git a/db/migrations/1756471290_updated_trails.go b/db/migrations/1756471290_updated_trails.go new file mode 100644 index 00000000..3c131c5c --- /dev/null +++ b/db/migrations/1756471290_updated_trails.go @@ -0,0 +1,42 @@ +package migrations + +import ( + "encoding/json" + + "github.com/pocketbase/pocketbase/core" + m "github.com/pocketbase/pocketbase/migrations" +) + +func init() { + m.Register(func(app core.App) error { + collection, err := app.FindCollectionByNameOrId("e864strfxo14pm4") + if err != nil { + return err + } + + // update collection data + if err := json.Unmarshal([]byte(`{ + "listRule": "author.user = @request.auth.id || public = true || (@request.auth.id != \"\" && trail_share_via_trail.actor.user ?= @request.auth.id) || (trail_link_share_via_trail.token != \"\" && trail_link_share_via_trail.token = @request.query.share)", + "viewRule": "author.user = @request.auth.id || public = true || (@request.auth.id != \"\" && trail_share_via_trail.actor.user ?= @request.auth.id) || (trail_link_share_via_trail.token != \"\" && trail_link_share_via_trail.token = @request.query.share)" + }`), &collection); err != nil { + return err + } + + return app.Save(collection) + }, func(app core.App) error { + collection, err := app.FindCollectionByNameOrId("e864strfxo14pm4") + if err != nil { + return err + } + + // update collection data + if err := json.Unmarshal([]byte(`{ + "listRule": "author.user = @request.auth.id || public = true || (@request.auth.id != \"\" && trail_share_via_trail.actor.user ?= @request.auth.id) || trail_link_share_via_trail.token = @request.query.share", + "viewRule": "author.user = @request.auth.id || public = true || (@request.auth.id != \"\" && trail_share_via_trail.actor.user ?= @request.auth.id) || trail_link_share_via_trail.token = @request.query.share " + }`), &collection); err != nil { + return err + } + + return app.Save(collection) + }) +} diff --git a/db/migrations/1756473199_updated_waypoints.go b/db/migrations/1756473199_updated_waypoints.go new file mode 100644 index 00000000..e8fca54f --- /dev/null +++ b/db/migrations/1756473199_updated_waypoints.go @@ -0,0 +1,42 @@ +package migrations + +import ( + "encoding/json" + + "github.com/pocketbase/pocketbase/core" + m "github.com/pocketbase/pocketbase/migrations" +) + +func init() { + m.Register(func(app core.App) error { + collection, err := app.FindCollectionByNameOrId("goeo2ubp103rzp9") + if err != nil { + return err + } + + // update collection data + if err := json.Unmarshal([]byte(`{ + "listRule": "author = @request.auth.id || trails_via_waypoints.author.user ?= @request.auth.id || trails_via_waypoints.public ?= true || \n(@collection.trail_share.trail.id ?= trails_via_waypoints.id && @collection.trail_share.actor.user ?= @request.auth.id)\n|| \n(@collection.trail_link_share.token != \"\" && @collection.trail_link_share.trail.waypoints.id ?= id)", + "viewRule": "author = @request.auth.id || trails_via_waypoints.author.user ?= @request.auth.id || trails_via_waypoints.public ?= true || \n(@collection.trail_share.trail.id ?= trails_via_waypoints.id && @collection.trail_share.actor.user ?= @request.auth.id)\n|| \n(@collection.trail_link_share.token != \"\" && @collection.trail_link_share.trail.waypoints.id ?= id)" + }`), &collection); err != nil { + return err + } + + return app.Save(collection) + }, func(app core.App) error { + collection, err := app.FindCollectionByNameOrId("goeo2ubp103rzp9") + if err != nil { + return err + } + + // update collection data + if err := json.Unmarshal([]byte(`{ + "listRule": "author = @request.auth.id || trails_via_waypoints.author.user ?= @request.auth.id || trails_via_waypoints.public ?= true || \n(@collection.trail_share.trail.id ?= trails_via_waypoints.id && @collection.trail_share.actor.user ?= @request.auth.id)\n|| \n(@collection.trail_link_share.trail.id ?= trails_via_waypoints.id && @collection.trail_link_share.token = @request.query.share)", + "viewRule": "author = @request.auth.id || trails_via_waypoints.author.user ?= @request.auth.id || trails_via_waypoints.public ?= true || \n(@collection.trail_share.trail.id ?= trails_via_waypoints.id && @collection.trail_share.actor.user ?= @request.auth.id)\n|| \n(@collection.trail_link_share.trail.id ?= trails_via_waypoints.id && @collection.trail_link_share.token = @request.query.share)" + }`), &collection); err != nil { + return err + } + + return app.Save(collection) + }) +} diff --git a/db/migrations/1757689107_updated_waypoints.go b/db/migrations/1757689107_updated_waypoints.go new file mode 100644 index 00000000..14e9e99c --- /dev/null +++ b/db/migrations/1757689107_updated_waypoints.go @@ -0,0 +1,62 @@ +package migrations + +import ( + "encoding/json" + + "github.com/pocketbase/pocketbase/core" + m "github.com/pocketbase/pocketbase/migrations" +) + +func init() { + m.Register(func(app core.App) error { + collection, err := app.FindCollectionByNameOrId("goeo2ubp103rzp9") + if err != nil { + return err + } + + // update collection data + if err := json.Unmarshal([]byte(`{ + "listRule": "author = @request.auth.id || trail.author.user ?= @request.auth.id || trail.public ?= true || trail.trail_share_via_trail.actor.user ?= @request.auth.id\n|| \n(trail.trail_link_share_via_trail.token != \"\" && trail.trail_link_share_via_trail.token = @request.query.share)", + "viewRule": "author = @request.auth.id || trail.author.user ?= @request.auth.id || trail.public ?= true || trail.trail_share_via_trail.actor.user ?= @request.auth.id\n|| \n(trail.trail_link_share_via_trail.token != \"\" && trail.trail_link_share_via_trail.token = @request.query.share)" + }`), &collection); err != nil { + return err + } + + // add field + if err := collection.Fields.AddMarshaledJSONAt(9, []byte(`{ + "cascadeDelete": true, + "collectionId": "e864strfxo14pm4", + "hidden": false, + "id": "relation2993194383", + "maxSelect": 1, + "minSelect": 0, + "name": "trail", + "presentable": false, + "required": true, + "system": false, + "type": "relation" + }`)); err != nil { + return err + } + + return app.Save(collection) + }, func(app core.App) error { + collection, err := app.FindCollectionByNameOrId("goeo2ubp103rzp9") + if err != nil { + return err + } + + // update collection data + if err := json.Unmarshal([]byte(`{ + "listRule": "author = @request.auth.id || trails_via_waypoints.author.user ?= @request.auth.id || trails_via_waypoints.public ?= true || \n(@collection.trail_share.trail.id ?= trails_via_waypoints.id && @collection.trail_share.actor.user ?= @request.auth.id)\n|| \n(@collection.trail_link_share.token != \"\" && @collection.trail_link_share.trail.waypoints.id ?= id)", + "viewRule": "author = @request.auth.id || trails_via_waypoints.author.user ?= @request.auth.id || trails_via_waypoints.public ?= true || \n(@collection.trail_share.trail.id ?= trails_via_waypoints.id && @collection.trail_share.actor.user ?= @request.auth.id)\n|| \n(@collection.trail_link_share.token != \"\" && @collection.trail_link_share.trail.waypoints.id ?= id)" + }`), &collection); err != nil { + return err + } + + // remove field + collection.Fields.RemoveById("relation2993194383") + + return app.Save(collection) + }) +} diff --git a/db/migrations/1757689185_migrate_waypoint_trails.go b/db/migrations/1757689185_migrate_waypoint_trails.go new file mode 100644 index 00000000..e92d2d1a --- /dev/null +++ b/db/migrations/1757689185_migrate_waypoint_trails.go @@ -0,0 +1,34 @@ +package migrations + +import ( + "github.com/pocketbase/dbx" + "github.com/pocketbase/pocketbase/core" + m "github.com/pocketbase/pocketbase/migrations" +) + +func init() { + m.Register(func(app core.App) error { + wps, err := app.FindAllRecords("waypoints") + if err != nil { + return err + } + + for _, wp := range wps { + trail, err := app.FindFirstRecordByFilter("trails", "waypoints ?~ {:id}", dbx.Params{"id": wp.Id}) + if err != nil { + continue + } + wp.Set("trail", trail.Id) + err = app.UnsafeWithoutHooks().Save(wp) + if err != nil { + return err + } + } + + return nil + }, func(app core.App) error { + // add down queries... + + return nil + }) +} diff --git a/db/migrations/1757690946_updated_trails.go b/db/migrations/1757690946_updated_trails.go new file mode 100644 index 00000000..d0cdeea7 --- /dev/null +++ b/db/migrations/1757690946_updated_trails.go @@ -0,0 +1,44 @@ +package migrations + +import ( + "github.com/pocketbase/pocketbase/core" + m "github.com/pocketbase/pocketbase/migrations" +) + +func init() { + m.Register(func(app core.App) error { + collection, err := app.FindCollectionByNameOrId("e864strfxo14pm4") + if err != nil { + return err + } + + // remove field + collection.Fields.RemoveById("ppq2sist") + + return app.Save(collection) + }, func(app core.App) error { + collection, err := app.FindCollectionByNameOrId("e864strfxo14pm4") + if err != nil { + return err + } + + // add field + if err := collection.Fields.AddMarshaledJSONAt(16, []byte(`{ + "cascadeDelete": false, + "collectionId": "goeo2ubp103rzp9", + "hidden": false, + "id": "ppq2sist", + "maxSelect": 2147483647, + "minSelect": 0, + "name": "waypoints", + "presentable": false, + "required": false, + "system": false, + "type": "relation" + }`)); err != nil { + return err + } + + return app.Save(collection) + }) +} diff --git a/db/migrations/1757759342_updated_activitypub_actors.go b/db/migrations/1757759342_updated_activitypub_actors.go new file mode 100644 index 00000000..40fd046a --- /dev/null +++ b/db/migrations/1757759342_updated_activitypub_actors.go @@ -0,0 +1,46 @@ +package migrations + +import ( + "encoding/json" + + "github.com/pocketbase/pocketbase/core" + m "github.com/pocketbase/pocketbase/migrations" +) + +func init() { + m.Register(func(app core.App) error { + collection, err := app.FindCollectionByNameOrId("pbc_1295301207") + if err != nil { + return err + } + + // update collection data + if err := json.Unmarshal([]byte(`{ + "indexes": [ + "CREATE UNIQUE INDEX `+"`"+`idx_rpT7QJwWTm`+"`"+` ON `+"`"+`activitypub_actors`+"`"+` (`+"`"+`iri`+"`"+`)", + "CREATE INDEX idx_actors_username_domain\nON activitypub_actors(preferred_username, domain);", + "CREATE INDEX idx_activitypub_actors_user ON activitypub_actors(user);" + ] + }`), &collection); err != nil { + return err + } + + return app.Save(collection) + }, func(app core.App) error { + collection, err := app.FindCollectionByNameOrId("pbc_1295301207") + if err != nil { + return err + } + + // update collection data + if err := json.Unmarshal([]byte(`{ + "indexes": [ + "CREATE UNIQUE INDEX `+"`"+`idx_rpT7QJwWTm`+"`"+` ON `+"`"+`activitypub_actors`+"`"+` (`+"`"+`iri`+"`"+`)" + ] + }`), &collection); err != nil { + return err + } + + return app.Save(collection) + }) +} diff --git a/db/util/activitypub.go b/db/util/activitypub.go index 197f25fd..b4c4f505 100644 --- a/db/util/activitypub.go +++ b/db/util/activitypub.go @@ -13,6 +13,7 @@ import ( "net/http" "net/url" "os" + "path" "strconv" "strings" "time" @@ -162,7 +163,19 @@ func TrailFromActivity(activity pub.Activity, app core.App, actor *core.Record) return nil, err } - record, err := app.FindFirstRecordByData("trails", "iri", t.ID.String()) + iri := t.ID.String() + var record *core.Record + if actor.GetBool(("isLocal")) { + trailUrl, parseErr := url.Parse(iri) + if parseErr != nil { + return nil, parseErr + } + trailId := path.Base(trailUrl.Path) + record, err = app.FindRecordById("trails", trailId) + } else { + record, err = app.FindFirstRecordByData("trails", "iri", iri) + } + if err != nil { if err == sql.ErrNoRows { collection, err := app.FindCollectionByNameOrId("trails") @@ -279,7 +292,7 @@ func TrailFromActivity(activity pub.Activity, app core.App, actor *core.Record) } if len(photoURLs) > 0 { - photos := make([]*filesystem.File, len(photoURLs)) + photos := []*filesystem.File{} for i, purl := range photoURLs { photo, err := filesystem.NewFileFromURL(context.Background(), purl) if err != nil { diff --git a/db/util/meilisearch.go b/db/util/meilisearch.go index a986909d..a2cb0f0e 100644 --- a/db/util/meilisearch.go +++ b/db/util/meilisearch.go @@ -69,7 +69,7 @@ func documentFromTrailRecord(app core.App, r *core.Record, author *core.Record, "elevation_gain": r.GetFloat("elevation_gain"), "elevation_loss": r.GetFloat("elevation_loss"), "duration": r.GetFloat("duration"), - "difficulty": r.Get("difficulty"), + "difficulty": difficultyToNumber(r.GetString("difficulty")), "category": category, "completed": logCount > 0, "date": r.GetDateTime("date").Time().Unix(), @@ -88,15 +88,52 @@ func documentFromTrailRecord(app core.App, r *core.Record, author *core.Record, } if includeShares { - document["shares"] = []string{} - document["likes"] = []string{} - document["like_count"] = 0 + trailShares := r.ExpandedAll("trail_share_via_trail") + if trailShares != nil { + sharedIDs := make([]string, len(trailShares)) + for i, v := range trailShares { + sharedIDs[i] = v.GetString("actor") + } + + document["shares"] = sharedIDs + + } else { + document["shares"] = []string{} + } + + trailLikes := r.ExpandedAll("trail_like_via_trail") + if trailLikes != nil { + likeIDs := make([]string, len(trailLikes)) + for i, v := range trailLikes { + likeIDs[i] = v.GetString("actor") + } + + document["likes"] = likeIDs + document["like_count"] = len(trailLikes) + + } else { + document["likes"] = []string{} + document["like_count"] = 0 + } } return document, nil } +func difficultyToNumber(difficulty string) int32 { + switch difficulty { + case "easy": + return 0 + case "moderate": + return 1 + case "difficult": + return 2 + } + + return 0 +} + func getPolyline(app core.App, r *core.Record) (string, error) { gpxPath := r.GetString("gpx") if len(gpxPath) == 0 { @@ -173,7 +210,7 @@ func documentFromListRecord(r *core.Record, author *core.Record, includeShares b domain = author.GetString("domain") } - document := map[string]interface{}{ + document := map[string]any{ "id": r.Id, "author": author.Id, "author_name": author.GetString("preferred_username"), @@ -193,7 +230,18 @@ func documentFromListRecord(r *core.Record, author *core.Record, includeShares b } if includeShares { - document["shares"] = []string{} + listShares := r.ExpandedAll("list_share_via_list") + if listShares != nil { + sharedIDs := make([]string, len(listShares)) + for i, v := range listShares { + sharedIDs[i] = v.GetString("actor") + } + + document["shares"] = sharedIDs + + } else { + document["shares"] = []string{} + } } return document, nil @@ -253,20 +301,40 @@ func documentFromRemoteRecord(r *core.Record, index string) (map[string]interfac return document, nil } -func IndexTrail(app core.App, r *core.Record, author *core.Record, client meilisearch.ServiceManager) error { - errs := app.ExpandRecord(r, []string{"tags"}, nil) - if len(errs) > 0 { - return fmt.Errorf("failed to expand tags: %v", errs) +func IndexTrails(app core.App, trails []*core.Record, client meilisearch.ServiceManager) error { + documents := make([]map[string]any, len(trails)) + + for i, r := range trails { + errs := app.ExpandRecord(r, []string{"tags"}, nil) + if len(errs) > 0 { + return fmt.Errorf("failed to expand tags: %v", errs) + } + errs = app.ExpandRecord(r, []string{"category"}, nil) + if len(errs) > 0 { + return fmt.Errorf("failed to expand category: %v", errs) + } + errs = app.ExpandRecord(r, []string{"trail_share_via_trail"}, nil) + if len(errs) > 0 { + return fmt.Errorf("failed to expand trail_share_via_trail: %v", errs) + } + errs = app.ExpandRecord(r, []string{"trail_like_via_trail"}, nil) + if len(errs) > 0 { + return fmt.Errorf("failed to expand trail_like_via_trail: %v", errs) + } + errs = app.ExpandRecord(r, []string{"author"}, nil) + if len(errs) > 0 { + return fmt.Errorf("failed to expand author: %v", errs) + } + + author := r.ExpandedOne("author") + + doc, err := documentFromTrailRecord(app, r, author, true) + if err != nil { + return err + } + + documents[i] = doc } - errs = app.ExpandRecord(r, []string{"category"}, nil) - if len(errs) > 0 { - return fmt.Errorf("failed to expand category: %v", errs) - } - doc, err := documentFromTrailRecord(app, r, author, true) - if err != nil { - return err - } - documents := []map[string]interface{}{doc} if _, err := client.Index("trails").AddDocuments(documents); err != nil { return err @@ -325,17 +393,32 @@ func UpdateTrailLikes(trailId string, likes []string, client meilisearch.Service return nil } -func IndexList(app core.App, r *core.Record, author *core.Record, client meilisearch.ServiceManager) error { - errs := app.ExpandRecord(r, []string{"trails"}, nil) - if len(errs) > 0 { - return fmt.Errorf("failed to expand trails: %v", errs) - } +func IndexLists(app core.App, lists []*core.Record, client meilisearch.ServiceManager) error { + documents := make([]map[string]any, len(lists)) - documents, err := documentFromListRecord(r, author, true) - if err != nil { - return err + for i, r := range lists { + errs := app.ExpandRecord(r, []string{"trails"}, nil) + if len(errs) > 0 { + return fmt.Errorf("failed to expand trails: %v", errs) + } + errs = app.ExpandRecord(r, []string{"list_share_via_list"}, nil) + if len(errs) > 0 { + return fmt.Errorf("failed to expand list_share_via_list: %v", errs) + } + errs = app.ExpandRecord(r, []string{"author"}, nil) + if len(errs) > 0 { + return fmt.Errorf("failed to expand author: %v", errs) + } + + author := r.ExpandedOne("author") + + doc, err := documentFromListRecord(r, author, true) + if err != nil { + return err + } + documents[i] = doc } - if _, err = client.Index("lists").AddDocuments(documents); err != nil { + if _, err := client.Index("lists").AddDocuments(documents); err != nil { return err } diff --git a/docs/src/content/docs/changelog.md b/docs/src/content/docs/changelog.md index 452e2d41..349805c4 100644 --- a/docs/src/content/docs/changelog.md +++ b/docs/src/content/docs/changelog.md @@ -2,6 +2,19 @@ title: Changelog description: What changed in the last patch? --- +## v0.18.2 +### Features +- Adds `dedup` command to pocketbase. This command allows an admin to quickly identify duplicate trails and delete them. Use the `--dry-run` flag to only log duplicate trails without deleting them. To execute the command run `docker exec -it wanderer-db ./pocketbase dedup --dry-run`. +- Adds option to only sync strava activities after a certain date +- Singificant performance improvements for instances with larger userbases +- Greatly improved initial indexing speed when starting wanderer +### Bug fixes +- Fixes permission issues for public trails +- Fixes bug that caused trails to be duplicated multiple times (to clean up see the `dedup` command above) +- Fixes link to "New Trail" from empty profiles +- Fixes link when opening a trail from the map searchbar +- Sorting by difficulty no longer sorts by difficulty alphabetically +- Fixes strava integration stopping after only one page ## v0.18.1 ### Bug fixes - Fixes permission issues that prevented federation from working properly diff --git a/web/src/hooks.server.ts b/web/src/hooks.server.ts index d7cf6c8c..ddf3e400 100644 --- a/web/src/hooks.server.ts +++ b/web/src/hooks.server.ts @@ -48,6 +48,7 @@ function isFormContentType(request: Request) { ); } +let publicMeilisearchKey: string | undefined = undefined; const auth: Handle = async ({ event, resolve }) => { const pb = new PocketBase(envPub.PUBLIC_POCKETBASE_URL) @@ -82,10 +83,14 @@ const auth: Handle = async ({ event, resolve }) => { if (pb.authStore.record) { meiliApiKey = pb.authStore.record.token settings = await pb.collection('settings').getFirstListItem(`user="${pb.authStore.record.id}"`, { requestKey: null }) - actor = await pb.collection("activitypub_actors").getFirstListItem(`user='${pb.authStore.record.id}'`) + actor = await pb.collection("activitypub_actors").getFirstListItem(`isLocal=1&&user='${pb.authStore.record.id}'`) } else { - const response = await pb.send("/public/search/token", { method: "GET", fetch: event.fetch }); - meiliApiKey = response.token; + if (!publicMeilisearchKey) { + const response = await pb.send("/public/search/token", { method: "GET", fetch: event.fetch }); + publicMeilisearchKey = response.token; + } + + meiliApiKey = publicMeilisearchKey!; } const ms = new MeiliSearch({ host: env.MEILI_URL, apiKey: meiliApiKey }); diff --git a/web/src/lib/components/settings/integrations/strava_settings_modal.svelte b/web/src/lib/components/settings/integrations/strava_settings_modal.svelte index b9a3c822..601d6afb 100644 --- a/web/src/lib/components/settings/integrations/strava_settings_modal.svelte +++ b/web/src/lib/components/settings/integrations/strava_settings_modal.svelte @@ -1,4 +1,5 @@ +

+ {$_("strava-integration-after-date-hint")} +

+
+ + +
{/snippet} {#snippet footer()} diff --git a/web/src/lib/components/trail/trail_filter_panel.svelte b/web/src/lib/components/trail/trail_filter_panel.svelte index 446bfc35..7979edfd 100644 --- a/web/src/lib/components/trail/trail_filter_panel.svelte +++ b/web/src/lib/components/trail/trail_filter_panel.svelte @@ -51,9 +51,9 @@ ]; const difficultyItems: SelectItem[] = [ - { text: $_("easy"), value: "easy" }, - { text: $_("moderate"), value: "moderate" }, - { text: $_("difficult"), value: "difficult" }, + { text: $_("easy"), value: 0 }, + { text: $_("moderate"), value: 1 }, + { text: $_("difficult"), value: 2 }, ]; let searchDropdownItems: SearchItem[] = $state([]); diff --git a/web/src/lib/components/trail/trail_info_panel.svelte b/web/src/lib/components/trail/trail_info_panel.svelte index 47ebdb4c..d674441b 100644 --- a/web/src/lib/components/trail/trail_info_panel.svelte +++ b/web/src/lib/components/trail/trail_info_panel.svelte @@ -658,7 +658,7 @@

{$_("start")}

- {#each trail.expand?.waypoints ?? [] as wp, i} + {#each trail.expand?.waypoints_via_trail ?? [] as wp, i}