yash
/
recipes2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update parser; minio bucket check

This commit is contained in:
yash 2024-01-21 13:52:21 +03:00
parent d7c777a2ca
commit 99a57be4c4
5 changed files with 125 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
cmd/parser/main.go
View File

@ -1 +1,79 @@
package main
import (
"context"
"log/slog"
"os"
"recipes/internal/config"
"recipes/internal/media_storage/minio"
"recipes/internal/parser"
"recipes/internal/storage/postgresql"
prettyLogger "github.com/charmbracelet/log"
)
const (
envLocal = "local"
envDev = "dev"
envProd = "prod"
)
func main() {
// read config
cfg := config.MustLoad()
// init logger
log := setupLogger(cfg.Env)
// init storage
storage, err := postgresql.New(
context.Background(),
cfg.Postgresql.User,
cfg.Postgresql.Password,
cfg.Postgresql.Address,
cfg.Postgresql.DBName,
)
if err != nil {
log.Error("failed to init storage", "err", err)
os.Exit(1)
}
// init media storage
mstorage, err := minio.New(
context.Background(),
cfg.Minio.Address,
cfg.Minio.User,
cfg.Minio.Password,
)
if err != nil {
log.Error("failed to init media storage", "err", err)
os.Exit(1)
}
// run parser
_, err = parser.SavePage(log, 1, mstorage, storage, storage)
if err != nil {
log.Error("Parse failed", "err", err)
os.Exit(1)
}
log.Info("parsing was completed successfully")
}
func setupLogger(env string) *slog.Logger {
var log *slog.Logger
switch env {
case envLocal:
// log = slog.New(
// slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug}),
// )
handler := prettyLogger.NewWithOptions(os.Stdout, prettyLogger.Options{Level: prettyLogger.DebugLevel})
log = slog.New(handler)
case envDev:
log = slog.New(
slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug}),
)
case envProd:
log = slog.New(
slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}),
)
}
return log
}

4
config/local.yaml
View File

@ -13,9 +13,9 @@ postgresql:
redis:
password: "password"
address: "127.0.0.1:6379"
address: "192.168.4.7:6379"
minio:
user: "user"
password: "password"
address: "127.0.0.1:9000"
address: "192.168.4.5:9000"

2
docker-compose.yml
View File

@ -64,7 +64,7 @@ services:
- MINIO_ROOT_USER=$MINIO_ROOT_USER
- MINIO_ROOT_PASSWORD=$MINIO_ROOT_PASSWORD
volumes:
- ../docker_data/recipes2_data/minio:/bitnami/minio/data
- ../docker_data/recipes2_data/minio:/bitnami/minio/data:z
ports:
- '9000:9000'
- '9001:9001'

61
internal/media_storage/minio/minio.go
View File

@ -4,7 +4,6 @@ import (
"context"
"fmt"
"io"
"log"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
@ -35,44 +34,14 @@ func New(ctx context.Context, addr, user, password string) (*ObjStorage, error)
return &ObjStorage{minio: minioClient}, nil
}
// // MinioConnection func for opening minio connection.
// func MinioConnection(bucketName string) (*minio.Client, error) {
// ctx := context.Background()
// useSSL := false
// // Initialize minio client object.
// minioClient, errInit := minio.New(config.Conf.MINIO_ADDR, &minio.Options{
// Creds: credentials.NewStaticV4(config.Conf.MINIO_ROOT_USER, config.Conf.MINIO_ROOT_PASSWORD, ""),
// Secure: useSSL,
// })
// if errInit != nil {
// return nil, errInit
// }
// // Check exists
// exists, errBucketExists := minioClient.BucketExists(ctx, bucketName)
// if errBucketExists != nil {
// return nil, errBucketExists
// }
// if !exists {
// // Create bucket
// err := minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: location})
// if err != nil {
// return nil, err
// } else {
// log.Printf("Successfully created %s\n", bucketName)
// }
// }
// return minioClient, errInit
// }
// Upload file to bucket
func (o *ObjStorage) uploadFile(ctx context.Context, bucketName string, objectName string, fileBuffer io.Reader, contentType string, fileSize int64) error {
const op = "media_storage.minio.UploadFile"
// Upload the zip file with PutObject
info, err := o.minio.PutObject(ctx, bucketName, objectName, fileBuffer, fileSize, minio.PutObjectOptions{ContentType: contentType})
_, err := o.minio.PutObject(ctx, bucketName, objectName, fileBuffer, fileSize, minio.PutObjectOptions{ContentType: contentType})
if err != nil {
return fmt.Errorf("%s: %w", op, err)
}
log.Printf("Successfully uploaded %s of size %d\n", objectName, info.Size)
return nil
}
@ -97,10 +66,32 @@ func (o *ObjStorage) delFile(ctx context.Context, bucketName string, objectName
return nil
}
// checkBucketExists creates bucket if it don't exists.
func (o *ObjStorage) checkBucketExists(ctx context.Context, bucketName string) error {
const op = "media_storage.minio.checkBucketExists"
// Check exists
exists, err := o.minio.BucketExists(ctx, bucketName)
if err != nil {
return fmt.Errorf("%s: %w", op, err)
}
if !exists {
// Create bucket
err := o.minio.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: location})
if err != nil {
return fmt.Errorf("%s: %w", op, err)
}
}
return nil
}
// SaveRecipeImage saves image to bucket for recipes photos.
func (o *ObjStorage) SaveRecipeImage(ctx context.Context, imageFile io.Reader, filename string, contentType string, fileSize int64) error {
const op = "media_storage.minio.SaveRecipeImage"
err := o.uploadFile(ctx, recipeImgBucket, filename, imageFile, contentType, fileSize)
err := o.checkBucketExists(ctx, recipeImgBucket)
if err != nil {
return fmt.Errorf("%s: %w", op, err)
}
err = o.uploadFile(ctx, recipeImgBucket, filename, imageFile, contentType, fileSize)
if err != nil {
return fmt.Errorf("%s: %w", op, err)
}
@ -110,6 +101,10 @@ func (o *ObjStorage) SaveRecipeImage(ctx context.Context, imageFile io.Reader, f
// RecipeImage gets image from recipe's images bucket by filename.
func (o *ObjStorage) RecipeImage(ctx context.Context, filename string) (*minio.Object, error) {
const op = "media_storage.minio.RecipeImage"
err := o.checkBucketExists(ctx, recipeImgBucket)
if err != nil {
return nil, fmt.Errorf("%s: %w", op, err)
}
obj, err := o.getFile(ctx, recipeImgBucket, filename)
if err != nil {
return nil, fmt.Errorf("%s: %w", op, err)

23
internal/parser/parser.go
View File

@ -47,7 +47,7 @@ type recipeProvider interface {
}
// SaveAllPages saves all pages to storage.
func SaveAllPages(log slog.Logger, ps pictureSaver, rs recipeSaver, rp recipeProvider) error {
func SaveAllPages(log *slog.Logger, ps pictureSaver, rs recipeSaver, rp recipeProvider) error {
const op = "parser.SaveAllPages"
// get total
log.Debug("Сохраняю страницу 1...")
@ -66,7 +66,7 @@ func SaveAllPages(log slog.Logger, ps pictureSaver, rs recipeSaver, rp recipePro
}
// SavePage saves page to storage.
func SavePage(log slog.Logger, page int, ps pictureSaver, rs recipeSaver, rp recipeProvider) (int, error) {
func SavePage(log *slog.Logger, page int, ps pictureSaver, rs recipeSaver, rp recipeProvider) (int, error) {
const op = "parser.SavePage"
var resp GetPageResp
@ -119,10 +119,14 @@ func SavePage(log slog.Logger, page int, ps pictureSaver, rs recipeSaver, rp rec
var wg sync.WaitGroup
wg.Add(len(recipes))
for i := 0; i < len(recipes); i++ {
go func(i int, log slog.Logger) {
go func(i int, log *slog.Logger) {
defer wg.Done()
err = GetRecipe(&recipes[i], ps, rs, rp)
if err != nil {
if errors.Is(err, RecipeExistsErr) {
log.Warn("Recipe already exists")
return
}
log.Error("Failed to get recipe", "err", fmt.Errorf("%s: %w", op, err))
}
}(i, log)
@ -225,11 +229,14 @@ func GetRecipe(r *models.Recipe, ps pictureSaver, rs recipeSaver, rp recipeProvi
// save picture
err = SaveRecipePicture(r, ps)
if err != nil {
return err
return fmt.Errorf("%s: %w", op, err)
}
// insert recipe
err = rs.AddRecipe(context.Background(), *r)
if err != nil {
return fmt.Errorf("%s: %w", op, err)
}
return nil
}
func SaveRecipePicture(r *models.Recipe, ps pictureSaver) error {
@ -254,7 +261,7 @@ func SaveRecipePicture(r *models.Recipe, ps pictureSaver) error {
}
// GetKey gets
func GetKey(log slog.Logger) error {
func GetKey(log *slog.Logger) error {
const op = "parser.GetKey"
log.Debug("Updating KEY...")
@ -267,14 +274,14 @@ func GetKey(log slog.Logger) error {
i := strings.Index(str, "\"key\":")
if i != 0 {
parseKey = str[i+7 : i+47]
log.Debug("New KEY =", parseKey)
log.Debug("New KEY", "key", parseKey)
return nil
}
return fmt.Errorf("%s: %w", op, KeyNotFoundErr)
}
func GetPHPSESSID(log slog.Logger) error {
func GetPHPSESSID(log *slog.Logger) error {
const op = "parser.GetPHPSESSID"
log.Debug("Updating PHPSESSID...")
@ -294,7 +301,7 @@ func GetPHPSESSID(log slog.Logger) error {
for _, c := range resp.Response().Cookies() {
if c.Name == "PHPSESSID" {
PHPSESSID = c.Value
log.Debug("New PHPSESSID =", PHPSESSID)
log.Debug("New PHPSESSID", "PHPSESSID", PHPSESSID)
return nil
}
}