From f34fd7afed3c3f78fa4f71190fe0bec38862164c Mon Sep 17 00:00:00 2001 From: Christian Beutel <> Date: Sun, 26 Jan 2025 17:28:38 +0100 Subject: [PATCH] adds list search --- .github/workflows/release.yaml | 3 - db/main.go | 106 ++++++++++++---- .../1737819003_meilisearch_add_lists_index.go | 117 ++++++++++++++++++ db/util/meilisearch.go | 51 ++++++++ docker-compose.yml | 1 + .../docs/getting-started/configuration.md | 1 + .../docs/getting-started/installation.mdx | 5 +- web/src/lib/components/base/search.svelte | 2 +- web/src/lib/components/nav_bar.svelte | 18 +-- web/src/lib/i18n/locales/de.json | 4 +- web/src/lib/i18n/locales/en.json | 2 +- web/src/lib/models/api/settings_schema.ts | 8 +- web/src/lib/models/settings.ts | 12 +- web/src/lib/stores/list_store.ts | 97 +++++++++++++-- web/src/lib/stores/search_store.ts | 12 +- web/src/lib/util/authorization_util.ts | 1 - web/src/routes/+page.svelte | 18 ++- .../routes/api/v1/search/nominatim/+server.ts | 17 --- web/src/routes/lists/+page.svelte | 12 +- web/src/routes/map/+page.svelte | 24 +++- 20 files changed, 413 insertions(+), 98 deletions(-) create mode 100644 db/migrations/1737819003_meilisearch_add_lists_index.go delete mode 100644 web/src/routes/api/v1/search/nominatim/+server.ts diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9161ca48..de671c3f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -78,9 +78,6 @@ jobs: env: VERSION: ${{ needs.versioning.outputs.version }} run: | - # Build search image - docker buildx build search/ --no-cache -t flomp/wanderer-search:$VERSION -t flomp/wanderer-search:latest --platform=linux/amd64,linux/arm64 --push - # Build db image cd db env GOOS=linux GOARCH=arm64 go build -o pocketbase_arm64 diff --git a/db/main.go b/db/main.go index d6799d7c..6ee312c6 100644 --- a/db/main.go +++ b/db/main.go @@ -61,9 +61,13 @@ func setupEventHandlers(app *pocketbase.PocketBase, client meilisearch.ServiceMa app.OnRecordAfterCreateRequest("trail_share").Add(createTrailShareHandler(app, client)) app.OnRecordAfterDeleteRequest("trail_share").Add(deleteTrailShareHandler(client)) - app.OnRecordAfterCreateRequest("list_share").Add(createListShareHandler(app)) + app.OnRecordAfterCreateRequest("lists").Add(createListHandler(app, client)) + app.OnRecordAfterUpdateRequest("lists").Add(updateListHandler(client)) + app.OnRecordAfterDeleteRequest("lists").Add(deleteListHandler(client)) + + app.OnRecordAfterCreateRequest("list_share").Add(createListShareHandler(app, client)) + app.OnRecordAfterDeleteRequest("list_share").Add(deleteListShareHandler(client)) - app.OnRecordAfterCreateRequest("lists").Add(createListHandler(app)) app.OnRecordAfterCreateRequest("follows").Add(createFollowHandler(app)) app.OnRecordAfterCreateRequest("comments").Add(createCommentHandler(app)) @@ -77,7 +81,9 @@ func createUserHandler(app *pocketbase.PocketBase, client meilisearch.ServiceMan userId := record.GetId() searchRules := map[string]interface{}{ - "cities500": map[string]string{}, + "lists": map[string]string{ + "filter": "public = true OR author = " + userId + " OR shares = " + userId, + }, "trails": map[string]string{ "filter": "public = true OR author = " + userId + " OR shares = " + userId, }, @@ -156,7 +162,11 @@ func createTrailShareHandler(app *pocketbase.PocketBase, client meilisearch.Serv for i, r := range shares { userIds[i] = r.GetString("user") } - util.UpdateTrailShares(trailId, userIds, client) + err = util.UpdateTrailShares(trailId, userIds, client) + + if err != nil { + return err + } if errs := app.Dao().ExpandRecord(e.Record, []string{"trail", "trail.author"}, nil); len(errs) > 0 { return fmt.Errorf("failed to expand: %v", errs) @@ -178,8 +188,66 @@ func createTrailShareHandler(app *pocketbase.PocketBase, client meilisearch.Serv } } -func createListShareHandler(app *pocketbase.PocketBase) func(e *core.RecordCreateEvent) error { +func deleteTrailShareHandler(client meilisearch.ServiceManager) func(e *core.RecordDeleteEvent) error { + return func(e *core.RecordDeleteEvent) error { + trailId := e.Record.GetString("trail") + return util.UpdateTrailShares(trailId, []string{}, client) + } +} + +func createListHandler(app *pocketbase.PocketBase, client meilisearch.ServiceManager) func(e *core.RecordCreateEvent) error { return func(e *core.RecordCreateEvent) error { + if err := util.IndexList(e.Record, client); err != nil { + return err + } + if !e.Record.GetBool("public") { + return nil + } + notification := util.Notification{ + Type: util.ListCreate, + Metadata: map[string]string{ + "id": e.Record.Id, + "list": e.Record.GetString("name"), + }, + Seen: false, + Author: e.Record.GetString("author"), + } + return util.SendNotificationToFollowers(app, notification) + } +} + +func updateListHandler(client meilisearch.ServiceManager) func(e *core.RecordUpdateEvent) error { + return func(e *core.RecordUpdateEvent) error { + return util.UpdateList(e.Record, client) + } +} + +func deleteListHandler(client meilisearch.ServiceManager) func(e *core.RecordDeleteEvent) error { + return func(e *core.RecordDeleteEvent) error { + _, err := client.Index("lists").DeleteDocument(e.Record.Id) + return err + } +} + +func createListShareHandler(app *pocketbase.PocketBase, client meilisearch.ServiceManager) func(e *core.RecordCreateEvent) error { + return func(e *core.RecordCreateEvent) error { + listId := e.Record.GetString("list") + shares, err := app.Dao().FindRecordsByExpr("list_share", + dbx.NewExp("list = {:listId}", dbx.Params{"listId": listId}), + ) + if err != nil { + return err + } + userIds := make([]string, len(shares)) + for i, r := range shares { + userIds[i] = r.GetString("user") + } + err = util.UpdateListShares(listId, userIds, client) + + if err != nil { + return err + } + if errs := app.Dao().ExpandRecord(e.Record, []string{"list", "list.author"}, nil); len(errs) > 0 { return fmt.Errorf("failed to expand: %v", errs) } @@ -200,28 +268,10 @@ func createListShareHandler(app *pocketbase.PocketBase) func(e *core.RecordCreat } } -func deleteTrailShareHandler(client meilisearch.ServiceManager) func(e *core.RecordDeleteEvent) error { +func deleteListShareHandler(client meilisearch.ServiceManager) func(e *core.RecordDeleteEvent) error { return func(e *core.RecordDeleteEvent) error { - trailId := e.Record.GetString("trail") - return util.UpdateTrailShares(trailId, []string{}, client) - } -} - -func createListHandler(app *pocketbase.PocketBase) func(e *core.RecordCreateEvent) error { - return func(e *core.RecordCreateEvent) error { - if !e.Record.GetBool("public") { - return nil - } - notification := util.Notification{ - Type: util.ListCreate, - Metadata: map[string]string{ - "id": e.Record.Id, - "list": e.Record.GetString("name"), - }, - Seen: false, - Author: e.Record.GetString("author"), - } - return util.SendNotificationToFollowers(app, notification) + listId := e.Record.GetString("list") + return util.UpdateListShares(listId, []string{}, client) } } @@ -293,7 +343,9 @@ func onBeforeServeHandler(app *pocketbase.PocketBase, client meilisearch.Service func registerRoutes(e *core.ServeEvent, app *pocketbase.PocketBase, client meilisearch.ServiceManager) { e.Router.GET("/public/search/token", func(c echo.Context) error { searchRules := map[string]interface{}{ - "cities500": map[string]string{}, + "lists": map[string]string{ + "filter": "public = true", + }, "trails": map[string]string{ "filter": "public = true", }, diff --git a/db/migrations/1737819003_meilisearch_add_lists_index.go b/db/migrations/1737819003_meilisearch_add_lists_index.go new file mode 100644 index 00000000..ab64653a --- /dev/null +++ b/db/migrations/1737819003_meilisearch_add_lists_index.go @@ -0,0 +1,117 @@ +package migrations + +import ( + "os" + "pocketbase/util" + + "github.com/meilisearch/meilisearch-go" + + "github.com/pocketbase/dbx" + "github.com/pocketbase/pocketbase/daos" + m "github.com/pocketbase/pocketbase/migrations" +) + +func init() { + client := meilisearch.New(os.Getenv("MEILI_URL"), meilisearch.WithAPIKey(os.Getenv("MEILI_MASTER_KEY"))) + + m.Register(func(db dbx.Builder) error { + + _, err := client.CreateIndex(&meilisearch.IndexConfig{ + Uid: "lists", + PrimaryKey: "id", + }) + if err != nil { + return err + } + + _, err = client.Index("lists").UpdateSortableAttributes(&[]string{ + "created", "name", + }) + if err != nil { + return err + } + + _, err = client.Index("lists").UpdateFilterableAttributes(&[]string{ + "author", "public", "shares", + }) + if err != nil { + return err + } + + dao := daos.New(db) + + lists, err := dao.FindRecordsByExpr("lists", dbx.NewExp("true")) + if err != nil { + return err + } + + for _, l := range lists { + err = util.IndexList(l, client) + if err != nil { + return err + } + shares, err := dao.FindRecordsByExpr("list_share", + dbx.NewExp("list = {:listId}", dbx.Params{"listId": l.Id}), + ) + if err != nil { + return err + } + userIds := make([]string, len(shares)) + for i, r := range shares { + userIds[i] = r.GetString("user") + } + err = util.UpdateListShares(l.Id, userIds, client) + if err != nil { + return err + } + } + + var usernames []string + err = db.NewQuery("SELECT username FROM users").Column(&usernames) + + if err != nil { + return err + } + + for _, username := range usernames { + + record, err := dao.FindAuthRecordByUsername("users", username) + if err != nil { + return err + } + + searchRules := map[string]interface{}{ + "lists": map[string]string{ + "filter": "public = true OR author = " + record.Id + " OR shares = " + record.Id, + }, + "trails": map[string]string{ + "filter": "public = true OR author = " + record.Id + " OR shares = " + record.Id, + }, + } + + if token, err := util.GenerateMeilisearchToken(searchRules, client); err != nil { + return err + } else { + record.Set("token", token) + + if err := dao.SaveRecord(record); err != nil { + return err + } + } + + } + + _, err = client.DeleteIndex("cities500") + if err != nil { + return err + } + + return nil + }, func(db dbx.Builder) error { + _, err := client.DeleteIndex("lists") + if err != nil { + return err + } + return nil + }) +} diff --git a/db/util/meilisearch.go b/db/util/meilisearch.go index 76068445..0b38ad87 100644 --- a/db/util/meilisearch.go +++ b/db/util/meilisearch.go @@ -38,6 +38,24 @@ func documentFromTrailRecord(r *models.Record, includeShares bool) map[string]in return document } +func documentFromListRecord(r *models.Record, includeShares bool) map[string]interface{} { + document := map[string]interface{}{ + "id": r.Id, + "author": r.GetString("author"), + "name": r.GetString("name"), + "description": r.GetString("description"), + "public": r.GetBool("public"), + "created": r.GetDateTime("created").Time().Unix(), + "trails": r.GetStringSlice("trails"), + } + + if includeShares { + document["shares"] = []string{} + } + + return document +} + func IndexTrail(r *models.Record, client meilisearch.ServiceManager) error { documents := []map[string]interface{}{documentFromTrailRecord(r, true)} @@ -71,6 +89,39 @@ func UpdateTrailShares(trailId string, shares []string, client meilisearch.Servi return nil } +func IndexList(r *models.Record, client meilisearch.ServiceManager) error { + documents := []map[string]interface{}{documentFromListRecord(r, true)} + + if _, err := client.Index("lists").AddDocuments(documents); err != nil { + return err + } + + return nil +} + +func UpdateList(r *models.Record, client meilisearch.ServiceManager) error { + documents := documentFromListRecord(r, false) + + if _, err := client.Index("lists").UpdateDocuments(documents); err != nil { + return err + } + + return nil +} + +func UpdateListShares(listId string, shares []string, client meilisearch.ServiceManager) error { + documents := []map[string]interface{}{ + { + "id": listId, + "shares": shares, + }, + } + if _, err := client.Index("lists").UpdateDocuments(documents); err != nil { + return err + } + return nil +} + func GenerateMeilisearchToken(rules map[string]interface{}, client meilisearch.ServiceManager) (resp string, err error) { apiKeyUid := "" apiKey := "" diff --git a/docker-compose.yml b/docker-compose.yml index 17dcae9a..dfdc7b31 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,6 +57,7 @@ services: UPLOAD_USER: UPLOAD_PASSWORD: PUBLIC_VALHALLA_URL: https://valhalla1.openstreetmap.de + PUBLIC_NOMINATIM_URL: https://nominatim.openstreetmap.org volumes: - ./data/uploads:/app/uploads ports: diff --git a/docs/src/content/docs/getting-started/configuration.md b/docs/src/content/docs/getting-started/configuration.md index e9868094..37084aa3 100644 --- a/docs/src/content/docs/getting-started/configuration.md +++ b/docs/src/content/docs/getting-started/configuration.md @@ -30,6 +30,7 @@ Since we use an unmodified installation of meilisearch you can use all variables | PUBLIC_POCKETBASE_URL | IP or hostname (including the port) of your wanderer instance | http://db:8090 | | PUBLIC_DISABLE_SIGNUP | Disables signup option for new users | false | | PUBLIC_VALHALLA_URL | Public IP or hostname (including the port) of a valhalla instance | https://valhalla1.openstreetmap.de | +| PUBLIC_NOMINATIM_URL | Public IP or hostname (including the port) of a nominatim instance | https://nominatim.openstreetmap.org| | UPLOAD_FOLDER | Folder from which wanderer auto-uploads trails | /app/uploads | | UPLOAD_USER | Username for the account with which wanderer auto-uploads trails | | | UPLOAD_PASSWORD | Password for the account with which wanderer auto-uploads trails | | diff --git a/docs/src/content/docs/getting-started/installation.mdx b/docs/src/content/docs/getting-started/installation.mdx index 04b59496..242fb219 100644 --- a/docs/src/content/docs/getting-started/installation.mdx +++ b/docs/src/content/docs/getting-started/installation.mdx @@ -79,6 +79,7 @@ services: UPLOAD_USER: UPLOAD_PASSWORD: PUBLIC_VALHALLA_URL: https://valhalla1.openstreetmap.de + PUBLIC_NOMINATIM_URL: https://nominatim.openstreetmap.org volumes: - ./data/uploads:/app/uploads ports: @@ -190,7 +191,3 @@ To update wanderer to the newest version simply run `git pull origin main` and r ## Verify the installation No matter which installation method you chose, you should now be able to access wanderer on localhost:3000. -:::note -On the first launch, wanderer will create a rather large city index with over 200,000 entries in meilisearch. This process happens automatically but can take up to 2 minutes to complete. During this time the search functionality might not yet work properly. -::: - diff --git a/web/src/lib/components/base/search.svelte b/web/src/lib/components/base/search.svelte index 9524ae7d..1911a272 100644 --- a/web/src/lib/components/base/search.svelte +++ b/web/src/lib/components/base/search.svelte @@ -119,7 +119,7 @@ {#if dropDownOpen}