From 99a57be4c4743be3ad6c49b0d7115687c7818ac8 Mon Sep 17 00:00:00 2001 From: yash Date: Sun, 21 Jan 2024 13:52:21 +0300 Subject: [PATCH] update parser; minio bucket check --- cmd/parser/main.go | 78 +++++++++++++++++++++++++++ config/local.yaml | 4 +- docker-compose.yml | 2 +- internal/media_storage/minio/minio.go | 61 ++++++++++----------- internal/parser/parser.go | 25 +++++---- 5 files changed, 125 insertions(+), 45 deletions(-) diff --git a/cmd/parser/main.go b/cmd/parser/main.go index 06ab7d0..acdd64b 100644 --- a/cmd/parser/main.go +++ b/cmd/parser/main.go @@ -1 +1,79 @@ package main + +import ( + "context" + "log/slog" + "os" + "recipes/internal/config" + "recipes/internal/media_storage/minio" + "recipes/internal/parser" + "recipes/internal/storage/postgresql" + + prettyLogger "github.com/charmbracelet/log" +) + +const ( + envLocal = "local" + envDev = "dev" + envProd = "prod" +) + +func main() { + // read config + cfg := config.MustLoad() + // init logger + log := setupLogger(cfg.Env) + // init storage + storage, err := postgresql.New( + context.Background(), + cfg.Postgresql.User, + cfg.Postgresql.Password, + cfg.Postgresql.Address, + cfg.Postgresql.DBName, + ) + if err != nil { + log.Error("failed to init storage", "err", err) + os.Exit(1) + } + // init media storage + mstorage, err := minio.New( + context.Background(), + cfg.Minio.Address, + cfg.Minio.User, + cfg.Minio.Password, + ) + if err != nil { + log.Error("failed to init media storage", "err", err) + os.Exit(1) + } + // run parser + _, err = parser.SavePage(log, 1, mstorage, storage, storage) + if err != nil { + log.Error("Parse failed", "err", err) + os.Exit(1) + } + log.Info("parsing was completed successfully") +} + +func setupLogger(env string) *slog.Logger { + var log *slog.Logger + + switch env { + case envLocal: + // log = slog.New( + // slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug}), + // ) + handler := prettyLogger.NewWithOptions(os.Stdout, prettyLogger.Options{Level: prettyLogger.DebugLevel}) + log = slog.New(handler) + case envDev: + log = slog.New( + slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug}), + ) + case envProd: + log = slog.New( + slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}), + ) + } + + return log +} diff --git a/config/local.yaml b/config/local.yaml index 7cd2644..a68f7b1 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -13,9 +13,9 @@ postgresql: redis: password: "password" - address: "127.0.0.1:6379" + address: "192.168.4.7:6379" minio: user: "user" password: "password" - address: "127.0.0.1:9000" + address: "192.168.4.5:9000" diff --git a/docker-compose.yml b/docker-compose.yml index 85517e7..f56f801 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,7 +64,7 @@ services: - MINIO_ROOT_USER=$MINIO_ROOT_USER - MINIO_ROOT_PASSWORD=$MINIO_ROOT_PASSWORD volumes: - - ../docker_data/recipes2_data/minio:/bitnami/minio/data + - ../docker_data/recipes2_data/minio:/bitnami/minio/data:z ports: - '9000:9000' - '9001:9001' diff --git a/internal/media_storage/minio/minio.go b/internal/media_storage/minio/minio.go index e2f904b..c814deb 100644 --- a/internal/media_storage/minio/minio.go +++ b/internal/media_storage/minio/minio.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" @@ -35,44 +34,14 @@ func New(ctx context.Context, addr, user, password string) (*ObjStorage, error) return &ObjStorage{minio: minioClient}, nil } -// // MinioConnection func for opening minio connection. -// func MinioConnection(bucketName string) (*minio.Client, error) { -// ctx := context.Background() -// useSSL := false -// // Initialize minio client object. -// minioClient, errInit := minio.New(config.Conf.MINIO_ADDR, &minio.Options{ -// Creds: credentials.NewStaticV4(config.Conf.MINIO_ROOT_USER, config.Conf.MINIO_ROOT_PASSWORD, ""), -// Secure: useSSL, -// }) -// if errInit != nil { -// return nil, errInit -// } -// // Check exists -// exists, errBucketExists := minioClient.BucketExists(ctx, bucketName) -// if errBucketExists != nil { -// return nil, errBucketExists -// } -// if !exists { -// // Create bucket -// err := minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: location}) -// if err != nil { -// return nil, err -// } else { -// log.Printf("Successfully created %s\n", bucketName) -// } -// } -// return minioClient, errInit -// } - // Upload file to bucket func (o *ObjStorage) uploadFile(ctx context.Context, bucketName string, objectName string, fileBuffer io.Reader, contentType string, fileSize int64) error { const op = "media_storage.minio.UploadFile" // Upload the zip file with PutObject - info, err := o.minio.PutObject(ctx, bucketName, objectName, fileBuffer, fileSize, minio.PutObjectOptions{ContentType: contentType}) + _, err := o.minio.PutObject(ctx, bucketName, objectName, fileBuffer, fileSize, minio.PutObjectOptions{ContentType: contentType}) if err != nil { return fmt.Errorf("%s: %w", op, err) } - log.Printf("Successfully uploaded %s of size %d\n", objectName, info.Size) return nil } @@ -97,10 +66,32 @@ func (o *ObjStorage) delFile(ctx context.Context, bucketName string, objectName return nil } +// checkBucketExists creates bucket if it don't exists. +func (o *ObjStorage) checkBucketExists(ctx context.Context, bucketName string) error { + const op = "media_storage.minio.checkBucketExists" + // Check exists + exists, err := o.minio.BucketExists(ctx, bucketName) + if err != nil { + return fmt.Errorf("%s: %w", op, err) + } + if !exists { + // Create bucket + err := o.minio.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: location}) + if err != nil { + return fmt.Errorf("%s: %w", op, err) + } + } + return nil +} + // SaveRecipeImage saves image to bucket for recipes photos. func (o *ObjStorage) SaveRecipeImage(ctx context.Context, imageFile io.Reader, filename string, contentType string, fileSize int64) error { const op = "media_storage.minio.SaveRecipeImage" - err := o.uploadFile(ctx, recipeImgBucket, filename, imageFile, contentType, fileSize) + err := o.checkBucketExists(ctx, recipeImgBucket) + if err != nil { + return fmt.Errorf("%s: %w", op, err) + } + err = o.uploadFile(ctx, recipeImgBucket, filename, imageFile, contentType, fileSize) if err != nil { return fmt.Errorf("%s: %w", op, err) } @@ -110,6 +101,10 @@ func (o *ObjStorage) SaveRecipeImage(ctx context.Context, imageFile io.Reader, f // RecipeImage gets image from recipe's images bucket by filename. func (o *ObjStorage) RecipeImage(ctx context.Context, filename string) (*minio.Object, error) { const op = "media_storage.minio.RecipeImage" + err := o.checkBucketExists(ctx, recipeImgBucket) + if err != nil { + return nil, fmt.Errorf("%s: %w", op, err) + } obj, err := o.getFile(ctx, recipeImgBucket, filename) if err != nil { return nil, fmt.Errorf("%s: %w", op, err) diff --git a/internal/parser/parser.go b/internal/parser/parser.go index 12fd125..7d9e9b2 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -47,7 +47,7 @@ type recipeProvider interface { } // SaveAllPages saves all pages to storage. -func SaveAllPages(log slog.Logger, ps pictureSaver, rs recipeSaver, rp recipeProvider) error { +func SaveAllPages(log *slog.Logger, ps pictureSaver, rs recipeSaver, rp recipeProvider) error { const op = "parser.SaveAllPages" // get total log.Debug("Сохраняю страницу 1...") @@ -66,7 +66,7 @@ func SaveAllPages(log slog.Logger, ps pictureSaver, rs recipeSaver, rp recipePro } // SavePage saves page to storage. -func SavePage(log slog.Logger, page int, ps pictureSaver, rs recipeSaver, rp recipeProvider) (int, error) { +func SavePage(log *slog.Logger, page int, ps pictureSaver, rs recipeSaver, rp recipeProvider) (int, error) { const op = "parser.SavePage" var resp GetPageResp @@ -119,10 +119,14 @@ func SavePage(log slog.Logger, page int, ps pictureSaver, rs recipeSaver, rp rec var wg sync.WaitGroup wg.Add(len(recipes)) for i := 0; i < len(recipes); i++ { - go func(i int, log slog.Logger) { + go func(i int, log *slog.Logger) { defer wg.Done() err = GetRecipe(&recipes[i], ps, rs, rp) if err != nil { + if errors.Is(err, RecipeExistsErr) { + log.Warn("Recipe already exists") + return + } log.Error("Failed to get recipe", "err", fmt.Errorf("%s: %w", op, err)) } }(i, log) @@ -225,11 +229,14 @@ func GetRecipe(r *models.Recipe, ps pictureSaver, rs recipeSaver, rp recipeProvi // save picture err = SaveRecipePicture(r, ps) if err != nil { - return err + return fmt.Errorf("%s: %w", op, err) } // insert recipe err = rs.AddRecipe(context.Background(), *r) - return fmt.Errorf("%s: %w", op, err) + if err != nil { + return fmt.Errorf("%s: %w", op, err) + } + return nil } func SaveRecipePicture(r *models.Recipe, ps pictureSaver) error { @@ -254,7 +261,7 @@ func SaveRecipePicture(r *models.Recipe, ps pictureSaver) error { } // GetKey gets -func GetKey(log slog.Logger) error { +func GetKey(log *slog.Logger) error { const op = "parser.GetKey" log.Debug("Updating KEY...") @@ -267,14 +274,14 @@ func GetKey(log slog.Logger) error { i := strings.Index(str, "\"key\":") if i != 0 { parseKey = str[i+7 : i+47] - log.Debug("New KEY =", parseKey) + log.Debug("New KEY", "key", parseKey) return nil } return fmt.Errorf("%s: %w", op, KeyNotFoundErr) } -func GetPHPSESSID(log slog.Logger) error { +func GetPHPSESSID(log *slog.Logger) error { const op = "parser.GetPHPSESSID" log.Debug("Updating PHPSESSID...") @@ -294,7 +301,7 @@ func GetPHPSESSID(log slog.Logger) error { for _, c := range resp.Response().Cookies() { if c.Name == "PHPSESSID" { PHPSESSID = c.Value - log.Debug("New PHPSESSID =", PHPSESSID) + log.Debug("New PHPSESSID", "PHPSESSID", PHPSESSID) return nil } }