From eecd78aba10502599916fa798b45abe643b59699 Mon Sep 17 00:00:00 2001 From: Christian Beutel <> Date: Wed, 13 Mar 2024 00:28:11 +0100 Subject: [PATCH] changes bootstrap mechanism --- db/Dockerfile | 3 +- db/go.mod | 4 +- db/main.go | 125 ++++++-------- db/meilisearch.go | 154 ++++++++++++++++++ docker-compose.yml | 30 +--- search/Dockerfile | 13 -- search/bootstrap.py | 65 -------- web/src/app.d.ts | 2 + web/src/hooks.server.ts | 18 +- web/src/lib/meilisearch.ts | 17 -- .../routes/api/v1/search/[index]/+server.ts | 3 +- web/src/routes/api/v1/search/multi/+server.ts | 3 +- 12 files changed, 232 insertions(+), 205 deletions(-) create mode 100644 db/meilisearch.go delete mode 100644 search/Dockerfile delete mode 100644 search/bootstrap.py delete mode 100644 web/src/lib/meilisearch.ts diff --git a/db/Dockerfile b/db/Dockerfile index 9e915dca..1e0c6f0a 100644 --- a/db/Dockerfile +++ b/db/Dockerfile @@ -6,14 +6,13 @@ WORKDIR / COPY go.mod ./ COPY go.sum ./ COPY migrations ./migrations +RUN tar -xvf migrations/initial_data/cities500.tar.gz -C migrations/initial_data RUN go mod download COPY *.go ./ RUN go build -o /pocketbase -RUN /pocketbase migrate -RUN /pocketbase seed ENV MEILI_URL=http://localhost:7700 ENV MEILI_MASTER_KEY= diff --git a/db/go.mod b/db/go.mod index 4351b342..3634e3e6 100644 --- a/db/go.mod +++ b/db/go.mod @@ -3,7 +3,9 @@ module pocketbase go 1.22.0 require ( + github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61 github.com/meilisearch/meilisearch-go v0.26.2 + github.com/pocketbase/dbx v1.10.1 github.com/pocketbase/pocketbase v0.21.3 ) @@ -50,13 +52,11 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.15.6 // indirect - github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-sqlite3 v1.14.19 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect - github.com/pocketbase/dbx v1.10.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/cobra v1.8.0 // indirect diff --git a/db/main.go b/db/main.go index 439df3fe..366c153f 100644 --- a/db/main.go +++ b/db/main.go @@ -1,11 +1,12 @@ package main import ( - "errors" "log" + "net/http" "os" "strings" + "github.com/labstack/echo/v5" "github.com/meilisearch/meilisearch-go" "github.com/pocketbase/pocketbase" "github.com/pocketbase/pocketbase/core" @@ -13,41 +14,10 @@ import ( "github.com/pocketbase/pocketbase/models" "github.com/pocketbase/pocketbase/plugins/migratecmd" "github.com/pocketbase/pocketbase/tools/filesystem" - "github.com/spf13/cobra" _ "pocketbase/migrations" ) -func indexRecord(r *models.Record, client *meilisearch.Client) error { - documents := []map[string]interface{}{ - { - "id": r.Id, - "author": r.GetString("author"), - "name": r.GetString("name"), - "description": r.GetString("description"), - "location": r.GetString("location"), - "distance": r.GetFloat("distance"), - "elevation_gain": r.GetFloat("elevation_gain"), - "duration": r.GetFloat("duration"), - "difficulty": r.Get("difficulty"), - "category": r.Get("category"), - "completed": len(r.GetStringSlice("summit_logs")) > 0, - "created": r.GetDateTime("created"), - "public": r.GetBool("public"), - "_geo": map[string]float64{ - "lat": r.GetFloat("lat"), - "lng": r.GetFloat("lon"), - }, - }, - } - - if _, err := client.Index("trails").AddDocuments(documents); err != nil { - return err - } - - return nil -} - func main() { app := pocketbase.New() @@ -56,26 +26,6 @@ func main() { Automigrate: true, }) - app.RootCmd.AddCommand(&cobra.Command{ - Use: "seed", - Run: func(cmd *cobra.Command, args []string) { - collection, _ := app.Dao().FindCollectionByNameOrId("categories") - - categories := []string{"Hiking", "Walking", "Climbing", "Skiing", "Canoeing", "Biking"} - for _, element := range categories { - record := models.NewRecord(collection) - form := forms.NewRecordUpsert(app, record) - form.LoadData(map[string]any{ - "name": element, - }) - f, _ := filesystem.NewFileFromPath("migrations/initial_data/" + strings.ToLower(element) + ".jpg") - form.AddFiles("img", f) - form.Submit() - } - - }, - }) - client := meilisearch.NewClient(meilisearch.ClientConfig{ Host: os.Getenv("MEILI_URL"), APIKey: os.Getenv("MEILI_MASTER_KEY"), @@ -84,35 +34,14 @@ func main() { app.OnRecordAfterCreateRequest("users").Add(func(e *core.RecordCreateEvent) error { userId := e.Record.GetId() - apiKeyUid := "" - apiKey := "" - - if keys, err := client.GetKeys(nil); err != nil { - log.Fatal(err) - } else { - for _, k := range keys.Results { - if k.Name == "Default Search API Key" { - apiKeyUid = k.UID - apiKey = k.Key - } - } - } - - if len(apiKey) == 0 || len(apiKeyUid) == 0 { - return errors.New("unable to locate meilisearch API key") - } - searchRules := map[string]interface{}{ "cities500": map[string]string{}, "trails": map[string]string{ "filter": "public = true OR author = " + userId, }, } - options := &meilisearch.TenantTokenOptions{ - APIKey: apiKey, - } - if token, err := client.GenerateTenantToken(apiKeyUid, searchRules, options); err != nil { + if token, err := generateMeilisearchToken(searchRules, client); err != nil { return err } else { e.Record.Set("token", token) @@ -146,6 +75,54 @@ func main() { return nil }) + app.OnBeforeServe().Add(func(e *core.ServeEvent) error { + e.Router.GET("/public/search/token", func(c echo.Context) error { + searchRules := map[string]interface{}{ + "cities500": map[string]string{}, + "trails": map[string]string{ + "filter": "public = true", + }, + } + + if token, err := generateMeilisearchToken(searchRules, client); err != nil { + return err + } else { + return c.JSON(http.StatusOK, map[string]string{"token": token}) + } + + }) + + // bootstrap pocketbase + query := app.Dao().RecordQuery("categories") + records := []*models.Record{} + + if err := query.All(&records); err != nil { + return err + } + if len(records) == 0 { + collection, _ := app.Dao().FindCollectionByNameOrId("categories") + + categories := []string{"Hiking", "Walking", "Climbing", "Skiing", "Canoeing", "Biking"} + for _, element := range categories { + record := models.NewRecord(collection) + form := forms.NewRecordUpsert(app, record) + form.LoadData(map[string]any{ + "name": element, + }) + f, _ := filesystem.NewFileFromPath("migrations/initial_data/" + strings.ToLower(element) + ".jpg") + form.AddFiles("img", f) + form.Submit() + } + } + + // bootstrap meilisearch + if err := bootstrapMeilisearch(client); err != nil { + return err + } + + return nil + }) + if err := app.Start(); err != nil { log.Fatal(err) } diff --git a/db/meilisearch.go b/db/meilisearch.go new file mode 100644 index 00000000..9ba4574b --- /dev/null +++ b/db/meilisearch.go @@ -0,0 +1,154 @@ +package main + +import ( + "encoding/json" + "errors" + "io" + "log" + "os" + + "github.com/meilisearch/meilisearch-go" + "github.com/pocketbase/pocketbase/models" +) + +func hasIndex(index string, client *meilisearch.Client) (resp bool, err error) { + + if _, err := client.GetIndex(index); err != nil { + if err.(*meilisearch.Error).StatusCode == 404 { + return false, nil + } + return false, err + } + return true, nil +} + +func bootstrapMeilisearch(client *meilisearch.Client) error { + indexExists, err := hasIndex("cities500", client) + if err != nil { + return err + } + if !indexExists { + + jsonFile, err := os.Open("migrations/initial_data/cities500.json") + + if err != nil { + return err + } + defer jsonFile.Close() + + byteValue, err := io.ReadAll(jsonFile) + + if err != nil { + return err + } + var cities []map[string]interface{} + json.Unmarshal(byteValue, &cities) + + _, err = client.CreateIndex(&meilisearch.IndexConfig{ + Uid: "cities500", + PrimaryKey: "id", + }) + if err != nil { + return err + } + settings := meilisearch.Settings{ + SortableAttributes: []string{ + "_geo", + }, + FilterableAttributes: []string{ + "_geo", + }, + } + _, err = client.Index("cities500").UpdateSettings(&settings) + if err != nil { + return err + } + _, err = client.Index("cities500").AddDocuments(cities) + if err != nil { + return (err) + } + } + + indexExists, err = hasIndex("trails", client) + if err != nil { + return err + } + if !indexExists { + _, err = client.CreateIndex(&meilisearch.IndexConfig{ + Uid: "trails", + PrimaryKey: "id", + }) + if err != nil { + return err + } + settings := meilisearch.Settings{ + SortableAttributes: []string{ + "name", "distance", "elevation_gain", "difficulty", "created", + }, + FilterableAttributes: []string{ + "category", "difficulty", "distance", "elevation_gain", "completed", "_geo", "public", "author", + }, + } + _, err = client.Index("trails").UpdateSettings(&settings) + if err != nil { + return err + } + } + return nil +} + +func indexRecord(r *models.Record, client *meilisearch.Client) error { + documents := []map[string]interface{}{ + { + "id": r.Id, + "author": r.GetString("author"), + "name": r.GetString("name"), + "description": r.GetString("description"), + "location": r.GetString("location"), + "distance": r.GetFloat("distance"), + "elevation_gain": r.GetFloat("elevation_gain"), + "duration": r.GetFloat("duration"), + "difficulty": r.Get("difficulty"), + "category": r.Get("category"), + "completed": len(r.GetStringSlice("summit_logs")) > 0, + "created": r.GetDateTime("created"), + "public": r.GetBool("public"), + "_geo": map[string]float64{ + "lat": r.GetFloat("lat"), + "lng": r.GetFloat("lon"), + }, + }, + } + + if _, err := client.Index("trails").AddDocuments(documents); err != nil { + return err + } + + return nil +} + +func generateMeilisearchToken(rules map[string]interface{}, client *meilisearch.Client) (resp string, err error) { + apiKeyUid := "" + apiKey := "" + + if keys, err := client.GetKeys(nil); err != nil { + log.Fatal(err) + } else { + for _, k := range keys.Results { + if k.Name == "Default Search API Key" { + apiKeyUid = k.UID + apiKey = k.Key + } + } + } + + if len(apiKey) == 0 || len(apiKeyUid) == 0 { + return "", errors.New("unable to locate meilisearch API key") + } + + options := &meilisearch.TenantTokenOptions{ + APIKey: apiKey, + } + + return client.GenerateTenantToken(apiKeyUid, rules, options) +} diff --git a/docker-compose.yml b/docker-compose.yml index 38df48cf..cde448d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ x-common-env: &cenv services: search: + container_name: wanderer-search image: getmeili/meilisearch:v1.6.2 environment: <<: *cenv @@ -18,50 +19,31 @@ services: volumes: - ./data/data.ms:/meili_data/data.ms restart: unless-stopped - - bootstrap: - build: ./search - environment: - <<: *cenv - depends_on: - search: - condition: service_started - networks: - - wanderer db: + container_name: wanderer-db build: ./db environment: <<: *cenv - depends_on: - bootstrap: - condition: service_completed_successfully ports: - "8090:8090" networks: - wanderer restart: unless-stopped volumes: - - wanderer-db:/pb_data + - ./data/pb_data:/pb_data web: + container_name: wanderer-web build: ./web environment: <<: *cenv - MEILI_API_TOKEN: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhcGlLZXlVaWQiOiIwZDA0YTAzYy1iNzBjLTQzZTctOGUzMC00NjNiODFhM2UwY2YiLCJzZWFyY2hSdWxlcyI6eyJ0cmFpbHMiOnsiZmlsdGVyIjoicHVibGljPXRydWUifSwiY2l0aWVzNTAwIjp7fX0sImV4cCI6bnVsbH0.XhHwyRqtKjij2-oPvcYAL3uIFmB4lxUBUgVzhQqL1To - ORIGIN: http://localhost:8080 + ORIGIN: http://localhost:3000 PUBLIC_POCKETBASE_URL: http://db:8090 ports: - - "8080:3000" - depends_on: - bootstrap: - condition: service_completed_successfully + - "3000:3000" networks: - wanderer restart: unless-stopped -volumes: - wanderer-db: - - networks: wanderer: driver: bridge diff --git a/search/Dockerfile b/search/Dockerfile deleted file mode 100644 index 3c216ce4..00000000 --- a/search/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM python:3.10-slim -WORKDIR /app - -COPY bootstrap.py /app -COPY cities500.tar.gz /app - -RUN tar -xvf cities500.tar.gz -RUN pip install meilisearch==0.30.0 - -ENV MEILI_URL=http://localhost:7700 -ENV MEILI_MASTER_KEY= - -CMD ["python", "-u", "./bootstrap.py"] \ No newline at end of file diff --git a/search/bootstrap.py b/search/bootstrap.py deleted file mode 100644 index a90f7a55..00000000 --- a/search/bootstrap.py +++ /dev/null @@ -1,65 +0,0 @@ -from meilisearch import Client -from meilisearch.errors import MeilisearchApiError -import json -import os - -MEILI_URL=os.getenv('MEILI_URL') -MEILI_MASTER_KEY=os.getenv('MEILI_MASTER_KEY') - -client = Client(MEILI_URL, MEILI_MASTER_KEY) - -def index_exists(index_name: str) -> bool: - try: - client.get_index(index_name) - return True - except MeilisearchApiError: - return False - -def init_indices(): - if not index_exists('cities500'): - print("Creating cities index...") - client.create_index('cities500', {'primaryKey': 'id'}) - - client.index('cities500').update_settings({ - 'sortableAttributes': ['_geo',], - 'filterableAttributes': ['_geo'] - }) - - print("Starting data import...") - json_file = open('cities500.json', encoding='utf-8') - cities = json.load(json_file) - - client.index('cities500').add_documents(cities) - print("Data import completed!") - - if not index_exists('trails'): - print("Creating trails index...") - client.create_index('trails', {'primaryKey': 'id'}) - - client.index('trails').update_settings({ - 'sortableAttributes': ['name', 'distance', 'elevation_gain', 'difficulty', 'created',], - 'filterableAttributes': ['category', 'difficulty', 'distance', 'elevation_gain', 'completed', '_geo', 'public', 'author'] - }) - - -def generate_public_token(): - search_key = next(filter(lambda r: r.name == "Default Search API Key", client.get_keys().results)) - - search_rules = { - 'trails': { - 'filter': 'public=true' - }, - 'cities500': {} - } - token = client.generate_tenant_token(api_key_uid=search_key.uid, search_rules=search_rules, api_key=search_key.key) - - return token - -print("Initializing indices...") -init_indices() -print("Indices initialized!") -token = generate_public_token() -print("Generating public token...") -print(f"MEILI_API_TOKEN: {token}") - -print("Bootstrapping complete!") diff --git a/web/src/app.d.ts b/web/src/app.d.ts index 30bb06c3..620fa86b 100644 --- a/web/src/app.d.ts +++ b/web/src/app.d.ts @@ -1,3 +1,4 @@ +import type MeiliSearch from 'meilisearch'; import PocketBase from 'pocketbase'; @@ -12,6 +13,7 @@ declare global { // interface Platform {} interface Locals { pb: PocketBase + ms: MeiliSearch user: AuthModel | null } } diff --git a/web/src/hooks.server.ts b/web/src/hooks.server.ts index c3d5968d..830459c3 100644 --- a/web/src/hooks.server.ts +++ b/web/src/hooks.server.ts @@ -1,7 +1,8 @@ -import { regenerateInstance } from '$lib/meilisearch' +import { env } from '$env/dynamic/private' import { pb } from '$lib/pocketbase' import { isRouteProtected } from '$lib/util/authorization_util' import { redirect, type Handle } from '@sveltejs/kit' +import { MeiliSearch } from 'meilisearch' import { locale } from 'svelte-i18n' export const handle: Handle = async ({ event, resolve }) => { @@ -12,13 +13,11 @@ export const handle: Handle = async ({ event, resolve }) => { // validate the user existence and if the path is acceesible if (!pb.authStore.model && isRouteProtected(url.pathname)) { - throw redirect(302, '/login?r='+url.pathname); + throw redirect(302, '/login?r=' + url.pathname); } else if (pb.authStore.model && url.pathname === "/login") { throw redirect(302, '/'); } - regenerateInstance(); - try { // get an up-to-date auth store state by verifying and refreshing the loaded auth model (if any) if (pb.authStore.isValid) { @@ -29,6 +28,17 @@ export const handle: Handle = async ({ event, resolve }) => { pb.authStore.clear() } + let meiliApiKey: string = ""; + if (pb.authStore.model) { + meiliApiKey = pb.authStore.model.token + } else { + const r = await event.fetch(pb.buildUrl("/public/search/token")); + const response = await r.json(); + meiliApiKey = response.token; + } + const ms = new MeiliSearch({ host: env.MEILI_URL, apiKey: meiliApiKey }); + + event.locals.ms = ms event.locals.pb = pb event.locals.user = pb.authStore.model diff --git a/web/src/lib/meilisearch.ts b/web/src/lib/meilisearch.ts deleted file mode 100644 index 832f3a51..00000000 --- a/web/src/lib/meilisearch.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { env } from '$env/dynamic/private' - -import { MeiliSearch } from 'meilisearch' -import { pb } from './pocketbase' - -export function createInstance() { - if (pb.authStore.model) { - return new MeiliSearch({ host: env.MEILI_URL, apiKey: pb.authStore.model.token }) - } - return new MeiliSearch({ host: env.MEILI_URL, apiKey: env.MEILI_API_TOKEN }) -} - -export function regenerateInstance() { - ms = createInstance(); -} - -export let ms = createInstance() \ No newline at end of file diff --git a/web/src/routes/api/v1/search/[index]/+server.ts b/web/src/routes/api/v1/search/[index]/+server.ts index 32e0e190..48e845c5 100644 --- a/web/src/routes/api/v1/search/[index]/+server.ts +++ b/web/src/routes/api/v1/search/[index]/+server.ts @@ -1,11 +1,10 @@ -import { ms } from "$lib/meilisearch"; import { error, json, type RequestEvent } from "@sveltejs/kit"; export async function POST(event: RequestEvent) { const data = await event.request.json() try { - const r = await ms.index(event.params.index as string).search(data.q, data.options); + const r = await event.locals.ms.index(event.params.index as string).search(data.q, data.options); return json(r); } catch (e: any) { console.log(e); diff --git a/web/src/routes/api/v1/search/multi/+server.ts b/web/src/routes/api/v1/search/multi/+server.ts index 4a43945f..952fff24 100644 --- a/web/src/routes/api/v1/search/multi/+server.ts +++ b/web/src/routes/api/v1/search/multi/+server.ts @@ -1,11 +1,10 @@ -import { ms } from "$lib/meilisearch"; import { error, json, type RequestEvent } from "@sveltejs/kit"; export async function POST(event: RequestEvent) { const data = await event.request.json() try { - const r = await ms.multiSearch({ + const r = await event.locals.ms.multiSearch({ queries: data.queries }); return json(r);