update parser; minio bucket check

cmd/parser/main.go

@@ -1 +1,79 @@
 package main
+
+import (
+	"context"
+	"log/slog"
+	"os"
+	"recipes/internal/config"
+	"recipes/internal/media_storage/minio"
+	"recipes/internal/parser"
+	"recipes/internal/storage/postgresql"
+
+	prettyLogger "github.com/charmbracelet/log"
+)
+
+const (
+	envLocal = "local"
+	envDev   = "dev"
+	envProd  = "prod"
+)
+
+func main() {
+	// read config
+	cfg := config.MustLoad()
+	// init logger
+	log := setupLogger(cfg.Env)
+	// init storage
+	storage, err := postgresql.New(
+		context.Background(),
+		cfg.Postgresql.User,
+		cfg.Postgresql.Password,
+		cfg.Postgresql.Address,
+		cfg.Postgresql.DBName,
+	)
+	if err != nil {
+		log.Error("failed to init storage", "err", err)
+		os.Exit(1)
+	}
+	// init media storage
+	mstorage, err := minio.New(
+		context.Background(),
+		cfg.Minio.Address,
+		cfg.Minio.User,
+		cfg.Minio.Password,
+	)
+	if err != nil {
+		log.Error("failed to init media storage", "err", err)
+		os.Exit(1)
+	}
+	// run parser
+	_, err = parser.SavePage(log, 1, mstorage, storage, storage)
+	if err != nil {
+		log.Error("Parse failed", "err", err)
+		os.Exit(1)
+	}
+	log.Info("parsing was completed successfully")
+}
+
+func setupLogger(env string) *slog.Logger {
+	var log *slog.Logger
+
+	switch env {
+	case envLocal:
+		// log = slog.New(
+		// 	slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug}),
+		// )
+		handler := prettyLogger.NewWithOptions(os.Stdout, prettyLogger.Options{Level: prettyLogger.DebugLevel})
+		log = slog.New(handler)
+	case envDev:
+		log = slog.New(
+			slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug}),
+		)
+	case envProd:
+		log = slog.New(
+			slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}),
+		)
+	}
+
+	return log
+}

config/local.yaml

@@ -13,9 +13,9 @@ postgresql:
 
 redis:
   password: "password"
-  address: "127.0.0.1:6379"
+  address: "192.168.4.7:6379"
 
 minio:
   user: "user"
   password: "password"
-  address: "127.0.0.1:9000"
+  address: "192.168.4.5:9000"

docker-compose.yml

@@ -64,7 +64,7 @@ services:
       - MINIO_ROOT_USER=$MINIO_ROOT_USER
       - MINIO_ROOT_PASSWORD=$MINIO_ROOT_PASSWORD
     volumes:
-      - ../docker_data/recipes2_data/minio:/bitnami/minio/data
+      - ../docker_data/recipes2_data/minio:/bitnami/minio/data:z
     ports:
       - '9000:9000'
       - '9001:9001'

internal/media_storage/minio/minio.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"log"
 
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
@@ -35,44 +34,14 @@ func New(ctx context.Context, addr, user, password string) (*ObjStorage, error)
 	return &ObjStorage{minio: minioClient}, nil
 }
 
-// // MinioConnection func for opening minio connection.
-// func MinioConnection(bucketName string) (*minio.Client, error) {
-// 	ctx := context.Background()
-// 	useSSL := false
-// 	// Initialize minio client object.
-// 	minioClient, errInit := minio.New(config.Conf.MINIO_ADDR, &minio.Options{
-// 		Creds:  credentials.NewStaticV4(config.Conf.MINIO_ROOT_USER, config.Conf.MINIO_ROOT_PASSWORD, ""),
-// 		Secure: useSSL,
-// 	})
-// 	if errInit != nil {
-// 		return nil, errInit
-// 	}
-// 	// Check exists
-// 	exists, errBucketExists := minioClient.BucketExists(ctx, bucketName)
-// 	if errBucketExists != nil {
-// 		return nil, errBucketExists
-// 	}
-// 	if !exists {
-// 		// Create bucket
-// 		err := minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: location})
-// 		if err != nil {
-// 			return nil, err
-// 		} else {
-// 			log.Printf("Successfully created %s\n", bucketName)
-// 		}
-// 	}
-// 	return minioClient, errInit
-// }
-
 // Upload file to bucket
 func (o *ObjStorage) uploadFile(ctx context.Context, bucketName string, objectName string, fileBuffer io.Reader, contentType string, fileSize int64) error {
 	const op = "media_storage.minio.UploadFile"
 	// Upload the zip file with PutObject
-	info, err := o.minio.PutObject(ctx, bucketName, objectName, fileBuffer, fileSize, minio.PutObjectOptions{ContentType: contentType})
+	_, err := o.minio.PutObject(ctx, bucketName, objectName, fileBuffer, fileSize, minio.PutObjectOptions{ContentType: contentType})
 	if err != nil {
 		return fmt.Errorf("%s: %w", op, err)
 	}
-	log.Printf("Successfully uploaded %s of size %d\n", objectName, info.Size)
 	return nil
 }
 
@@ -97,10 +66,32 @@ func (o *ObjStorage) delFile(ctx context.Context, bucketName string, objectName
 	return nil
 }
 
+// checkBucketExists creates bucket if it don't exists.
+func (o *ObjStorage) checkBucketExists(ctx context.Context, bucketName string) error {
+	const op = "media_storage.minio.checkBucketExists"
+	// Check exists
+	exists, err := o.minio.BucketExists(ctx, bucketName)
+	if err != nil {
+		return fmt.Errorf("%s: %w", op, err)
+	}
+	if !exists {
+		// Create bucket
+		err := o.minio.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: location})
+		if err != nil {
+			return fmt.Errorf("%s: %w", op, err)
+		}
+	}
+	return nil
+}
+
 // SaveRecipeImage saves image to bucket for recipes photos.
 func (o *ObjStorage) SaveRecipeImage(ctx context.Context, imageFile io.Reader, filename string, contentType string, fileSize int64) error {
 	const op = "media_storage.minio.SaveRecipeImage"
-	err := o.uploadFile(ctx, recipeImgBucket, filename, imageFile, contentType, fileSize)
+	err := o.checkBucketExists(ctx, recipeImgBucket)
+	if err != nil {
+		return fmt.Errorf("%s: %w", op, err)
+	}
+	err = o.uploadFile(ctx, recipeImgBucket, filename, imageFile, contentType, fileSize)
 	if err != nil {
 		return fmt.Errorf("%s: %w", op, err)
 	}
@@ -110,6 +101,10 @@ func (o *ObjStorage) SaveRecipeImage(ctx context.Context, imageFile io.Reader, f
 // RecipeImage gets image from recipe's images bucket by filename.
 func (o *ObjStorage) RecipeImage(ctx context.Context, filename string) (*minio.Object, error) {
 	const op = "media_storage.minio.RecipeImage"
+	err := o.checkBucketExists(ctx, recipeImgBucket)
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", op, err)
+	}
 	obj, err := o.getFile(ctx, recipeImgBucket, filename)
 	if err != nil {
 		return nil, fmt.Errorf("%s: %w", op, err)

internal/parser/parser.go

@@ -47,7 +47,7 @@ type recipeProvider interface {
 }
 
 // SaveAllPages saves all pages to storage.
-func SaveAllPages(log slog.Logger, ps pictureSaver, rs recipeSaver, rp recipeProvider) error {
+func SaveAllPages(log *slog.Logger, ps pictureSaver, rs recipeSaver, rp recipeProvider) error {
 	const op = "parser.SaveAllPages"
 	// get total
 	log.Debug("Сохраняю страницу 1...")
@@ -66,7 +66,7 @@ func SaveAllPages(log slog.Logger, ps pictureSaver, rs recipeSaver, rp recipePro
 }
 
 // SavePage saves page to storage.
-func SavePage(log slog.Logger, page int, ps pictureSaver, rs recipeSaver, rp recipeProvider) (int, error) {
+func SavePage(log *slog.Logger, page int, ps pictureSaver, rs recipeSaver, rp recipeProvider) (int, error) {
 	const op = "parser.SavePage"
 
 	var resp GetPageResp
@@ -119,10 +119,14 @@ func SavePage(log slog.Logger, page int, ps pictureSaver, rs recipeSaver, rp rec
 	var wg sync.WaitGroup
 	wg.Add(len(recipes))
 	for i := 0; i < len(recipes); i++ {
-		go func(i int, log slog.Logger) {
+		go func(i int, log *slog.Logger) {
 			defer wg.Done()
 			err = GetRecipe(&recipes[i], ps, rs, rp)
 			if err != nil {
+				if errors.Is(err, RecipeExistsErr) {
+					log.Warn("Recipe already exists")
+					return
+				}
 				log.Error("Failed to get recipe", "err", fmt.Errorf("%s: %w", op, err))
 			}
 		}(i, log)
@@ -225,12 +229,15 @@ func GetRecipe(r *models.Recipe, ps pictureSaver, rs recipeSaver, rp recipeProvi
 	// save picture
 	err = SaveRecipePicture(r, ps)
 	if err != nil {
-		return err
+		return fmt.Errorf("%s: %w", op, err)
 	}
 	// insert recipe
 	err = rs.AddRecipe(context.Background(), *r)
+	if err != nil {
 		return fmt.Errorf("%s: %w", op, err)
 	}
+	return nil
+}
 
 func SaveRecipePicture(r *models.Recipe, ps pictureSaver) error {
 	const op = "parser.SaveRecipePicture"
@@ -254,7 +261,7 @@ func SaveRecipePicture(r *models.Recipe, ps pictureSaver) error {
 }
 
 // GetKey gets
-func GetKey(log slog.Logger) error {
+func GetKey(log *slog.Logger) error {
 	const op = "parser.GetKey"
 
 	log.Debug("Updating KEY...")
@@ -267,14 +274,14 @@ func GetKey(log slog.Logger) error {
 	i := strings.Index(str, "\"key\":")
 	if i != 0 {
 		parseKey = str[i+7 : i+47]
-		log.Debug("New KEY =", parseKey)
+		log.Debug("New KEY", "key", parseKey)
 		return nil
 	}
 
 	return fmt.Errorf("%s: %w", op, KeyNotFoundErr)
 }
 
-func GetPHPSESSID(log slog.Logger) error {
+func GetPHPSESSID(log *slog.Logger) error {
 	const op = "parser.GetPHPSESSID"
 
 	log.Debug("Updating PHPSESSID...")
@@ -294,7 +301,7 @@ func GetPHPSESSID(log slog.Logger) error {
 	for _, c := range resp.Response().Cookies() {
 		if c.Name == "PHPSESSID" {
 			PHPSESSID = c.Value
-			log.Debug("New PHPSESSID =", PHPSESSID)
+			log.Debug("New PHPSESSID", "PHPSESSID", PHPSESSID)
 			return nil
 		}
 	}