fix komoot sync stopping before all tours are imported (#917)

Three bugs caused partial imports:

1. Type filter halted pagination: if a page contained only tours of the
   excluded type (planned vs. recorded), syncTrailWithTours returned false
   and the loop stopped even though later pages might have matching tours.
   Fixed by tracking allAlreadySynced separately - only tours already in the
   DB count toward early termination, not type-filtered ones.

2. continue on error never incremented page: a fetch or sync error called
   continue on the outer for loop, so page stayed the same and hasNewTours
   was never updated, causing an infinite retry on the failing page. Changed
   to break so a failed page aborts the current sync cycle cleanly.

3. Pagination ignored TotalPages: the loop used a fragile stop heuristic
   instead of the TotalPages value returned by the Komoot API. fetchTours
   now returns TotalPages and the loop uses it as the upper bound, with the
   allAlreadySynced early-exit kept as an optimization for incremental syncs.

Closes #570

Co-authored-by: slothful-vassal <89943360+slothful-vassal@users.noreply.github.com>
This commit is contained in:
Stefan Schlögl
2026-04-20 20:24:51 +02:00
committed by GitHub
parent 871bf38112
commit 19fd259129

View File

@@ -68,25 +68,27 @@ func SyncKomoot(app core.App) error {
app.Logger().Warn(warning) app.Logger().Warn(warning)
continue continue
} }
hasNewTours := true totalPages := 1
page := 0 for page := 0; page < totalPages; page++ {
for hasNewTours { tours, tp, err := k.fetchTours(page)
tours, err := k.fetchTours(page)
if err != nil { if err != nil {
warning := fmt.Sprintf("error fetching tours from komoot: %v\n", err) warning := fmt.Sprintf("error fetching tours from komoot (page %d): %v\n", page, err)
fmt.Print(warning) fmt.Print(warning)
app.Logger().Warn(warning) app.Logger().Warn(warning)
continue break
} }
totalPages = tp
hasNewTours, err = syncTrailWithTours(app, k, komootIntegration, userId, actorId, tours) allAlreadySynced, err := syncTrailWithTours(app, k, komootIntegration, userId, actorId, tours)
if err != nil { if err != nil {
warning := fmt.Sprintf("error syncing komoot tours with trails: %v\n", err) warning := fmt.Sprintf("error syncing komoot tours with trails: %v\n", err)
fmt.Print(warning) fmt.Print(warning)
app.Logger().Warn(warning) app.Logger().Warn(warning)
continue break
}
if allAlreadySynced {
break
} }
page += 1
} }
} }
@@ -156,20 +158,18 @@ func (k *KomootApi) Login(email, password string) error {
return nil return nil
} }
func (k *KomootApi) fetchTours(page int) ([]KomootTour, error) { func (k *KomootApi) fetchTours(page int) ([]KomootTour, int, error) {
currentUri := fmt.Sprintf("https://api.komoot.de/v007/users/%s/tours/?page=%d&sort_field=date&sort_direction=desc&limit=30", k.UserID, page) currentUri := fmt.Sprintf("https://api.komoot.de/v007/users/%s/tours/?page=%d&sort_field=date&sort_direction=desc&limit=30", k.UserID, page)
body, err := sendRequest(currentUri, k.buildHeader()) body, err := sendRequest(currentUri, k.buildHeader())
if err != nil { if err != nil {
return nil, err return nil, 0, err
} }
var data KomootToursResponse var data KomootToursResponse
json.Unmarshal(body, &data) json.Unmarshal(body, &data)
tours := data.Embedded.Tours return data.Embedded.Tours, data.Page.TotalPages, nil
return tours, nil
} }
func (k *KomootApi) fetchDetailedTour(tour KomootTour) (*DetailedKomootTour, error) { func (k *KomootApi) fetchDetailedTour(tour KomootTour) (*DetailedKomootTour, error) {
@@ -184,17 +184,25 @@ func (k *KomootApi) fetchDetailedTour(tour KomootTour) (*DetailedKomootTour, err
return data, nil return data, nil
} }
// syncTrailWithTours imports tours not yet in the DB. Returns allAlreadySynced=true
// when every tour on this page was already imported, so the caller can stop paginating
// early during incremental syncs. Tours skipped due to type filters do NOT count as
// synced - only tours already present in the DB do.
func syncTrailWithTours(app core.App, k *KomootApi, i KomootIntegration, user string, actor string, tours []KomootTour) (bool, error) { func syncTrailWithTours(app core.App, k *KomootApi, i KomootIntegration, user string, actor string, tours []KomootTour) (bool, error) {
hasNewTours := false allAlreadySynced := true
for _, tour := range tours { for _, tour := range tours {
trails, err := app.FindRecordsByFilter("trails", "external_id = {:id}", "", 1, 0, dbx.Params{"id": strconv.Itoa(int(tour.ID))}) trails, err := app.FindRecordsByFilter("trails", "external_id = {:id}", "", 1, 0, dbx.Params{"id": strconv.Itoa(int(tour.ID))})
if err != nil { if err != nil {
return hasNewTours, err return false, err
} }
if len(trails) != 0 || (tour.Type == "tour_planned" && !i.Planned) || (tour.Type == "tour_recorded" && !i.Completed) { if len(trails) != 0 {
continue
}
// Tour is not yet in the DB - we must keep paginating regardless of type filter
allAlreadySynced = false
if (tour.Type == "tour_planned" && !i.Planned) || (tour.Type == "tour_recorded" && !i.Completed) {
continue continue
} }
hasNewTours = true
detailedTour, err := k.fetchDetailedTour(tour) detailedTour, err := k.fetchDetailedTour(tour)
if err != nil { if err != nil {
app.Logger().Warn(fmt.Sprintf("Unable to fetch details for tour '%s': %v", tour.Name, err)) app.Logger().Warn(fmt.Sprintf("Unable to fetch details for tour '%s': %v", tour.Name, err))
@@ -217,7 +225,7 @@ func syncTrailWithTours(app core.App, k *KomootApi, i KomootIntegration, user st
} }
} }
return hasNewTours, nil return allAlreadySynced, nil
} }
func createTrailFromTour(app core.App, k *KomootApi, detailedTour *DetailedKomootTour, gpx *filesystem.File, user string, actor string, privacy string) (string, error) { func createTrailFromTour(app core.App, k *KomootApi, detailedTour *DetailedKomootTour, gpx *filesystem.File, user string, actor string, privacy string) (string, error) {