add cache

This commit is contained in:
yyasha 2024-01-27 21:42:23 +03:00
parent 665a469f27
commit 090efaf84a
9 changed files with 211 additions and 28 deletions

15
go.mod
View File

@ -3,17 +3,22 @@ module recipes
go 1.21.5
require (
github.com/PuerkitoBio/goquery v1.8.1
github.com/charmbracelet/log v0.3.1
github.com/go-chi/chi/v5 v5.0.11
github.com/go-chi/render v1.0.3
github.com/go-playground/validator/v10 v10.17.0
github.com/golang-migrate/migrate/v4 v4.17.0
github.com/google/uuid v1.5.0
github.com/ilyakaznacheev/cleanenv v1.5.0
github.com/jackc/pgx/v5 v5.5.2
github.com/minio/minio-go/v7 v7.0.66
github.com/redis/go-redis/v9 v9.4.0
github.com/s32x/httpclient v0.0.0-20220217184346-6df4d4d51c14
)
require (
github.com/BurntSushi/toml v1.2.1 // indirect
github.com/PuerkitoBio/goquery v1.8.1 // indirect
github.com/ajg/form v1.5.1 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
@ -23,14 +28,9 @@ require (
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/go-chi/chi/v5 v5.0.11 // indirect
github.com/go-chi/render v1.0.3 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-logfmt/logfmt v0.6.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.17.0 // indirect
github.com/google/uuid v1.5.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
@ -47,9 +47,7 @@ require (
github.com/mattn/go-isatty v0.0.18 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/minio-go v6.0.14+incompatible // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/muesli/reflow v0.3.0 // indirect
@ -57,7 +55,6 @@ require (
github.com/rivo/uniseg v0.2.0 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/s32x/httpclient v0.0.0-20220217184346-6df4d4d51c14 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
go.uber.org/atomic v1.7.0 // indirect
golang.org/x/crypto v0.17.0 // indirect

10
go.sum
View File

@ -48,10 +48,10 @@ github.com/go-chi/chi/v5 v5.0.11 h1:BnpYbFZ3T3S1WMpD79r7R5ThWX40TaFB7L31Y8xqSwA=
github.com/go-chi/chi/v5 v5.0.11/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4=
github.com/go-chi/render v1.0.3/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
@ -65,6 +65,7 @@ github.com/golang-migrate/migrate/v4 v4.17.0/go.mod h1:+Cp2mtLP4/aXDTKb9wmXYitdr
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/h12w/go-socks5 v0.0.0-20200522160539-76189e178364 h1:5XxdakFhqd9dnXoAZy1Mb2R/DZ6D1e+0bGC/JhucGYI=
github.com/h12w/go-socks5 v0.0.0-20200522160539-76189e178364/go.mod h1:eDJQioIyy4Yn3MVivT7rv/39gAJTrA7lgmYr8EW950c=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
@ -107,14 +108,10 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o=
github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8=
github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw=
github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs=
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@ -132,6 +129,7 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

View File

@ -4,6 +4,7 @@ import (
"context"
"log/slog"
httpsrv "recipes/internal/app/httpSrv"
"recipes/internal/cache_provider/redis"
"recipes/internal/config"
"recipes/internal/media_storage/minio"
"recipes/internal/storage/postgresql"
@ -19,13 +20,15 @@ func New(log *slog.Logger, cfg *config.Config) *App {
if err != nil {
panic(err)
}
// init cache
cache := redis.New(cfg.Redis.Address, cfg.Redis.Password, storage)
// init media storage
mstorage, err := minio.New(context.Background(), cfg.Minio.Address, cfg.Minio.User, cfg.Minio.Password)
if err != nil {
panic(err)
}
// init http server
httpsrv := httpsrv.New(log, &cfg.HTTPServerConfig, storage, mstorage)
httpsrv := httpsrv.New(log, &cfg.HTTPServerConfig, cache, mstorage)
// return
return &App{HTTPSrv: httpsrv}
}

View File

@ -1,13 +1,198 @@
package redis
import (
"context"
"encoding/json"
"errors"
"fmt"
"recipes/internal/domain/models"
"time"
"github.com/redis/go-redis/v9"
)
type Cache struct {
rdb *redis.Client
type Storage interface {
AddRecipe(ctx context.Context, recipe models.Recipe) error
RecipeExists(ctx context.Context, title string) (bool, error)
GetRecipes(ctx context.Context, offset, limit int) ([]models.Recipe, error)
GetRecipe(ctx context.Context, r_id uint) (models.Recipe, error)
GetRecipesByCategory(ctx context.Context, offset, limit int, category string) ([]models.Recipe, error)
}
func New() *Cache {
return &Cache{}
type CacheProvider struct {
rdb *redis.Client
storage Storage
}
const cacheExpiration = time.Hour
func New(addr, password string, s Storage) *CacheProvider {
rdb := redis.NewClient(&redis.Options{
Addr: addr,
Password: password,
})
return &CacheProvider{
rdb: rdb,
storage: s,
}
}
// GetRecipe gets recipe from cache (if exists) or from storage.
func (c *CacheProvider) GetRecipe(ctx context.Context, r_id uint) (models.Recipe, error) {
const op = "cache_provider.redis.GetRecipe"
// try to get from cache
cache_data := c.rdb.Get(ctx, fmt.Sprintf("recipe:%d", r_id))
if cache_data.Err() != nil {
if errors.Is(cache_data.Err(), redis.Nil) {
fmt.Println("--- STORAGE ---")
// if empty - get from storage
recipe, err := c.addRecipe(ctx, r_id)
if err != nil {
return models.Recipe{}, fmt.Errorf("%s: %w", op, err)
}
return recipe, nil
}
return models.Recipe{}, fmt.Errorf("%s: %w", op, cache_data.Err())
}
fmt.Println("--- CACHE ---")
// decode
var recipe models.Recipe
err := json.Unmarshal([]byte(cache_data.Val()), &recipe)
if err != nil {
return models.Recipe{}, fmt.Errorf("%s: %w", op, err)
}
// return
return recipe, nil
}
// addRecipe adds recipe from storage to cache and returned it.
func (c *CacheProvider) addRecipe(ctx context.Context, r_id uint) (models.Recipe, error) {
const op = "cache_provider.redis.addRecipe"
// Get from storage
recipe, err := c.storage.GetRecipe(ctx, r_id)
if err != nil {
return models.Recipe{}, fmt.Errorf("%s: %w", op, err)
}
// Save to cache
key := fmt.Sprintf("recipe:%d", recipe.ID)
// if _, err := c.rdb.Pipelined(ctx, func(p redis.Pipeliner) error {
// p.HSet(ctx, key, "title", recipe.Title)
// p.HSet(ctx, key, "desc", recipe.Description)
// p.HSet(ctx, key, "img", recipe.Image)
// p.HSet(ctx, key, "ctime", recipe.CookingTime)
// p.HSet(ctx, key, "snum", recipe.ServingsNum)
// p.HSet(ctx, key, "cal", recipe.Calories)
// return nil
// }); err != nil {
// return models.Recipe{}, fmt.Errorf("%s: %w", op, err)
// }
// encode
cache_data, err := json.Marshal(recipe)
if err != nil {
return models.Recipe{}, fmt.Errorf("%s: %w", op, err)
}
s := c.rdb.Set(ctx, key, cache_data, cacheExpiration)
if s.Err() != nil {
return models.Recipe{}, fmt.Errorf("%s: %w", op, err)
}
return recipe, nil
}
// GetRecipes gets recipes from cache if exists or from storage.
func (c *CacheProvider) GetRecipes(ctx context.Context, offset, limit int) ([]models.Recipe, error) {
const op = "cache_provider.redis.GetRecipes"
// try to get from cache
key := fmt.Sprintf("recipes_page:%d:%d", offset, limit)
cache_data := c.rdb.Get(ctx, key)
if cache_data.Err() != nil {
if errors.Is(cache_data.Err(), redis.Nil) {
fmt.Println("--- STORAGE ---")
// get from storage
recipes, err := c.addRecipesPage(ctx, offset, limit)
if err != nil {
return nil, fmt.Errorf("%s: %w", op, err)
}
return recipes, nil
}
return nil, fmt.Errorf("%s: %w", op, cache_data.Err())
}
fmt.Println("--- CACHE ---")
// decode
var recipes []models.Recipe = make([]models.Recipe, 0, limit)
err := json.Unmarshal([]byte(cache_data.Val()), &recipes)
if err != nil {
return nil, fmt.Errorf("%s: %w", op, err)
}
return recipes, nil
}
// addRecipesPage adds recipes list (page) to cache.
func (c *CacheProvider) addRecipesPage(ctx context.Context, offset, limit int) ([]models.Recipe, error) {
const op = "cache_provider.redis.addRecipesPage"
// Get from storage
recipes, err := c.storage.GetRecipes(ctx, offset, limit)
if err != nil {
return nil, fmt.Errorf("%s: %w", op, err)
}
// Save to cache
cache_data, err := json.Marshal(recipes)
if err != nil {
return nil, fmt.Errorf("%s: %w", op, err)
}
key := fmt.Sprintf("recipes_page:%d:%d", offset, limit)
s := c.rdb.Set(ctx, key, cache_data, cacheExpiration)
if s.Err() != nil {
return nil, fmt.Errorf("%s: %w", op, err)
}
return recipes, nil
}
// GetRecipesByCategory gets recipes by category from cache if exists or from storage if not.
func (c *CacheProvider) GetRecipesByCategory(ctx context.Context, offset, limit int, category string) ([]models.Recipe, error) {
const op = "cache_provider.redis.GetRecipesByCategory"
// try to get from cache
key := fmt.Sprintf("r:%s:%d:%d", category, offset, limit)
cache_data := c.rdb.Get(ctx, key)
if cache_data.Err() != nil {
if errors.Is(cache_data.Err(), redis.Nil) {
fmt.Println("--- STORAGE ---")
// get from storage
recipes, err := c.addRecipesByCategory(ctx, offset, limit, category)
if err != nil {
return nil, fmt.Errorf("%s: %w", op, err)
}
return recipes, nil
}
return nil, fmt.Errorf("%s: %w", op, cache_data.Err())
}
fmt.Println("--- CACHE ---")
// decode
var recipes []models.Recipe = make([]models.Recipe, 0, limit)
err := json.Unmarshal([]byte(cache_data.Val()), &recipes)
if err != nil {
return nil, fmt.Errorf("%s: %w", op, err)
}
return recipes, nil
}
func (c *CacheProvider) addRecipesByCategory(ctx context.Context, offset, limit int, category string) ([]models.Recipe, error) {
const op = "cache_provider.redis.addRecipesByCategory"
// Get from storage
recipes, err := c.storage.GetRecipesByCategory(ctx, offset, limit, category)
if err != nil {
return nil, fmt.Errorf("%s: %w", op, err)
}
// Save to cache
cache_data, err := json.Marshal(recipes)
if err != nil {
return nil, fmt.Errorf("%s: %w", op, err)
}
key := fmt.Sprintf("r:%s:%d:%d", category, offset, limit)
s := c.rdb.Set(ctx, key, cache_data, cacheExpiration)
if s.Err() != nil {
return nil, fmt.Errorf("%s: %w", op, err)
}
return recipes, nil
}

View File

@ -32,7 +32,7 @@ func New(log *slog.Logger, recipeProvider RecipeProvider) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
const op = "http-server.handlers.recipe.New"
log = log.With(
log := log.With(
slog.String("op", op),
slog.String("request_id", middleware.GetReqID(r.Context())),
)

View File

@ -21,7 +21,7 @@ func New(log *slog.Logger, imageProvider ImageProvider) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
const op = "http-server.handlers.recipeImage.New"
log = log.With(
log := log.With(
slog.String("op", op),
slog.String("request_id", middleware.GetReqID(r.Context())),
)

View File

@ -32,7 +32,7 @@ func New(log *slog.Logger, recipesProvider RecipesProvider) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
const op = "http-server.handlers.recipes.New"
log = log.With(
log := log.With(
slog.String("op", op),
slog.String("request_id", middleware.GetReqID(r.Context())),
)

View File

@ -15,7 +15,7 @@ import (
type Request struct {
Page uint `json:"page" validate:"required,gt=0"`
Category string `json:"category" validate:"required,containsany"`
Category string `json:"category" validate:"required"`
}
type Response struct {
@ -33,7 +33,7 @@ func New(log *slog.Logger, recipesProvider RecipesProvider) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
const op = "http-server.handlers.recipesByCategory.New"
log = log.With(
log := log.With(
slog.String("op", op),
slog.String("request_id", middleware.GetReqID(r.Context())),
)

View File

@ -193,7 +193,7 @@ func (s *Storage) GetRecipe(ctx context.Context, r_id uint) (models.Recipe, erro
return models.Recipe{}, fmt.Errorf("%s: %w", op, err)
}
err = s.AddRecipeInformation(ctx, &recipe)
err = s.addRecipeInformation(ctx, &recipe)
if err != nil {
return models.Recipe{}, fmt.Errorf("%s: %w", op, err)
}
@ -201,8 +201,8 @@ func (s *Storage) GetRecipe(ctx context.Context, r_id uint) (models.Recipe, erro
return recipe, nil
}
// AddRecipeInformation adds to recipe struct info about ingredients, steps, advices, categories.
func (s *Storage) AddRecipeInformation(ctx context.Context, r *models.Recipe) error {
// addRecipeInformation adds to recipe struct info about ingredients, steps, advices, categories.
func (s *Storage) addRecipeInformation(ctx context.Context, r *models.Recipe) error {
const op = "storage.postgresql.AddRecipeInformation"
// select ingredients