update parser; minio bucket check
This commit is contained in:
parent
d7c777a2ca
commit
99a57be4c4
|
@ -1 +1,79 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"recipes/internal/config"
|
||||||
|
"recipes/internal/media_storage/minio"
|
||||||
|
"recipes/internal/parser"
|
||||||
|
"recipes/internal/storage/postgresql"
|
||||||
|
|
||||||
|
prettyLogger "github.com/charmbracelet/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
envLocal = "local"
|
||||||
|
envDev = "dev"
|
||||||
|
envProd = "prod"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
// read config
|
||||||
|
cfg := config.MustLoad()
|
||||||
|
// init logger
|
||||||
|
log := setupLogger(cfg.Env)
|
||||||
|
// init storage
|
||||||
|
storage, err := postgresql.New(
|
||||||
|
context.Background(),
|
||||||
|
cfg.Postgresql.User,
|
||||||
|
cfg.Postgresql.Password,
|
||||||
|
cfg.Postgresql.Address,
|
||||||
|
cfg.Postgresql.DBName,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("failed to init storage", "err", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
// init media storage
|
||||||
|
mstorage, err := minio.New(
|
||||||
|
context.Background(),
|
||||||
|
cfg.Minio.Address,
|
||||||
|
cfg.Minio.User,
|
||||||
|
cfg.Minio.Password,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("failed to init media storage", "err", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
// run parser
|
||||||
|
_, err = parser.SavePage(log, 1, mstorage, storage, storage)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Parse failed", "err", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
log.Info("parsing was completed successfully")
|
||||||
|
}
|
||||||
|
|
||||||
|
func setupLogger(env string) *slog.Logger {
|
||||||
|
var log *slog.Logger
|
||||||
|
|
||||||
|
switch env {
|
||||||
|
case envLocal:
|
||||||
|
// log = slog.New(
|
||||||
|
// slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug}),
|
||||||
|
// )
|
||||||
|
handler := prettyLogger.NewWithOptions(os.Stdout, prettyLogger.Options{Level: prettyLogger.DebugLevel})
|
||||||
|
log = slog.New(handler)
|
||||||
|
case envDev:
|
||||||
|
log = slog.New(
|
||||||
|
slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug}),
|
||||||
|
)
|
||||||
|
case envProd:
|
||||||
|
log = slog.New(
|
||||||
|
slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return log
|
||||||
|
}
|
||||||
|
|
|
@ -13,9 +13,9 @@ postgresql:
|
||||||
|
|
||||||
redis:
|
redis:
|
||||||
password: "password"
|
password: "password"
|
||||||
address: "127.0.0.1:6379"
|
address: "192.168.4.7:6379"
|
||||||
|
|
||||||
minio:
|
minio:
|
||||||
user: "user"
|
user: "user"
|
||||||
password: "password"
|
password: "password"
|
||||||
address: "127.0.0.1:9000"
|
address: "192.168.4.5:9000"
|
||||||
|
|
|
@ -64,7 +64,7 @@ services:
|
||||||
- MINIO_ROOT_USER=$MINIO_ROOT_USER
|
- MINIO_ROOT_USER=$MINIO_ROOT_USER
|
||||||
- MINIO_ROOT_PASSWORD=$MINIO_ROOT_PASSWORD
|
- MINIO_ROOT_PASSWORD=$MINIO_ROOT_PASSWORD
|
||||||
volumes:
|
volumes:
|
||||||
- ../docker_data/recipes2_data/minio:/bitnami/minio/data
|
- ../docker_data/recipes2_data/minio:/bitnami/minio/data:z
|
||||||
ports:
|
ports:
|
||||||
- '9000:9000'
|
- '9000:9000'
|
||||||
- '9001:9001'
|
- '9001:9001'
|
||||||
|
|
|
@ -4,7 +4,6 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
|
||||||
|
|
||||||
"github.com/minio/minio-go/v7"
|
"github.com/minio/minio-go/v7"
|
||||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||||
|
@ -35,44 +34,14 @@ func New(ctx context.Context, addr, user, password string) (*ObjStorage, error)
|
||||||
return &ObjStorage{minio: minioClient}, nil
|
return &ObjStorage{minio: minioClient}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// // MinioConnection func for opening minio connection.
|
|
||||||
// func MinioConnection(bucketName string) (*minio.Client, error) {
|
|
||||||
// ctx := context.Background()
|
|
||||||
// useSSL := false
|
|
||||||
// // Initialize minio client object.
|
|
||||||
// minioClient, errInit := minio.New(config.Conf.MINIO_ADDR, &minio.Options{
|
|
||||||
// Creds: credentials.NewStaticV4(config.Conf.MINIO_ROOT_USER, config.Conf.MINIO_ROOT_PASSWORD, ""),
|
|
||||||
// Secure: useSSL,
|
|
||||||
// })
|
|
||||||
// if errInit != nil {
|
|
||||||
// return nil, errInit
|
|
||||||
// }
|
|
||||||
// // Check exists
|
|
||||||
// exists, errBucketExists := minioClient.BucketExists(ctx, bucketName)
|
|
||||||
// if errBucketExists != nil {
|
|
||||||
// return nil, errBucketExists
|
|
||||||
// }
|
|
||||||
// if !exists {
|
|
||||||
// // Create bucket
|
|
||||||
// err := minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: location})
|
|
||||||
// if err != nil {
|
|
||||||
// return nil, err
|
|
||||||
// } else {
|
|
||||||
// log.Printf("Successfully created %s\n", bucketName)
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// return minioClient, errInit
|
|
||||||
// }
|
|
||||||
|
|
||||||
// Upload file to bucket
|
// Upload file to bucket
|
||||||
func (o *ObjStorage) uploadFile(ctx context.Context, bucketName string, objectName string, fileBuffer io.Reader, contentType string, fileSize int64) error {
|
func (o *ObjStorage) uploadFile(ctx context.Context, bucketName string, objectName string, fileBuffer io.Reader, contentType string, fileSize int64) error {
|
||||||
const op = "media_storage.minio.UploadFile"
|
const op = "media_storage.minio.UploadFile"
|
||||||
// Upload the zip file with PutObject
|
// Upload the zip file with PutObject
|
||||||
info, err := o.minio.PutObject(ctx, bucketName, objectName, fileBuffer, fileSize, minio.PutObjectOptions{ContentType: contentType})
|
_, err := o.minio.PutObject(ctx, bucketName, objectName, fileBuffer, fileSize, minio.PutObjectOptions{ContentType: contentType})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("%s: %w", op, err)
|
return fmt.Errorf("%s: %w", op, err)
|
||||||
}
|
}
|
||||||
log.Printf("Successfully uploaded %s of size %d\n", objectName, info.Size)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -97,10 +66,32 @@ func (o *ObjStorage) delFile(ctx context.Context, bucketName string, objectName
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// checkBucketExists creates bucket if it don't exists.
|
||||||
|
func (o *ObjStorage) checkBucketExists(ctx context.Context, bucketName string) error {
|
||||||
|
const op = "media_storage.minio.checkBucketExists"
|
||||||
|
// Check exists
|
||||||
|
exists, err := o.minio.BucketExists(ctx, bucketName)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%s: %w", op, err)
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
// Create bucket
|
||||||
|
err := o.minio.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: location})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%s: %w", op, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// SaveRecipeImage saves image to bucket for recipes photos.
|
// SaveRecipeImage saves image to bucket for recipes photos.
|
||||||
func (o *ObjStorage) SaveRecipeImage(ctx context.Context, imageFile io.Reader, filename string, contentType string, fileSize int64) error {
|
func (o *ObjStorage) SaveRecipeImage(ctx context.Context, imageFile io.Reader, filename string, contentType string, fileSize int64) error {
|
||||||
const op = "media_storage.minio.SaveRecipeImage"
|
const op = "media_storage.minio.SaveRecipeImage"
|
||||||
err := o.uploadFile(ctx, recipeImgBucket, filename, imageFile, contentType, fileSize)
|
err := o.checkBucketExists(ctx, recipeImgBucket)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%s: %w", op, err)
|
||||||
|
}
|
||||||
|
err = o.uploadFile(ctx, recipeImgBucket, filename, imageFile, contentType, fileSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("%s: %w", op, err)
|
return fmt.Errorf("%s: %w", op, err)
|
||||||
}
|
}
|
||||||
|
@ -110,6 +101,10 @@ func (o *ObjStorage) SaveRecipeImage(ctx context.Context, imageFile io.Reader, f
|
||||||
// RecipeImage gets image from recipe's images bucket by filename.
|
// RecipeImage gets image from recipe's images bucket by filename.
|
||||||
func (o *ObjStorage) RecipeImage(ctx context.Context, filename string) (*minio.Object, error) {
|
func (o *ObjStorage) RecipeImage(ctx context.Context, filename string) (*minio.Object, error) {
|
||||||
const op = "media_storage.minio.RecipeImage"
|
const op = "media_storage.minio.RecipeImage"
|
||||||
|
err := o.checkBucketExists(ctx, recipeImgBucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%s: %w", op, err)
|
||||||
|
}
|
||||||
obj, err := o.getFile(ctx, recipeImgBucket, filename)
|
obj, err := o.getFile(ctx, recipeImgBucket, filename)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%s: %w", op, err)
|
return nil, fmt.Errorf("%s: %w", op, err)
|
||||||
|
|
|
@ -47,7 +47,7 @@ type recipeProvider interface {
|
||||||
}
|
}
|
||||||
|
|
||||||
// SaveAllPages saves all pages to storage.
|
// SaveAllPages saves all pages to storage.
|
||||||
func SaveAllPages(log slog.Logger, ps pictureSaver, rs recipeSaver, rp recipeProvider) error {
|
func SaveAllPages(log *slog.Logger, ps pictureSaver, rs recipeSaver, rp recipeProvider) error {
|
||||||
const op = "parser.SaveAllPages"
|
const op = "parser.SaveAllPages"
|
||||||
// get total
|
// get total
|
||||||
log.Debug("Сохраняю страницу 1...")
|
log.Debug("Сохраняю страницу 1...")
|
||||||
|
@ -66,7 +66,7 @@ func SaveAllPages(log slog.Logger, ps pictureSaver, rs recipeSaver, rp recipePro
|
||||||
}
|
}
|
||||||
|
|
||||||
// SavePage saves page to storage.
|
// SavePage saves page to storage.
|
||||||
func SavePage(log slog.Logger, page int, ps pictureSaver, rs recipeSaver, rp recipeProvider) (int, error) {
|
func SavePage(log *slog.Logger, page int, ps pictureSaver, rs recipeSaver, rp recipeProvider) (int, error) {
|
||||||
const op = "parser.SavePage"
|
const op = "parser.SavePage"
|
||||||
|
|
||||||
var resp GetPageResp
|
var resp GetPageResp
|
||||||
|
@ -119,10 +119,14 @@ func SavePage(log slog.Logger, page int, ps pictureSaver, rs recipeSaver, rp rec
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
wg.Add(len(recipes))
|
wg.Add(len(recipes))
|
||||||
for i := 0; i < len(recipes); i++ {
|
for i := 0; i < len(recipes); i++ {
|
||||||
go func(i int, log slog.Logger) {
|
go func(i int, log *slog.Logger) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
err = GetRecipe(&recipes[i], ps, rs, rp)
|
err = GetRecipe(&recipes[i], ps, rs, rp)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if errors.Is(err, RecipeExistsErr) {
|
||||||
|
log.Warn("Recipe already exists")
|
||||||
|
return
|
||||||
|
}
|
||||||
log.Error("Failed to get recipe", "err", fmt.Errorf("%s: %w", op, err))
|
log.Error("Failed to get recipe", "err", fmt.Errorf("%s: %w", op, err))
|
||||||
}
|
}
|
||||||
}(i, log)
|
}(i, log)
|
||||||
|
@ -225,11 +229,14 @@ func GetRecipe(r *models.Recipe, ps pictureSaver, rs recipeSaver, rp recipeProvi
|
||||||
// save picture
|
// save picture
|
||||||
err = SaveRecipePicture(r, ps)
|
err = SaveRecipePicture(r, ps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return fmt.Errorf("%s: %w", op, err)
|
||||||
}
|
}
|
||||||
// insert recipe
|
// insert recipe
|
||||||
err = rs.AddRecipe(context.Background(), *r)
|
err = rs.AddRecipe(context.Background(), *r)
|
||||||
return fmt.Errorf("%s: %w", op, err)
|
if err != nil {
|
||||||
|
return fmt.Errorf("%s: %w", op, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func SaveRecipePicture(r *models.Recipe, ps pictureSaver) error {
|
func SaveRecipePicture(r *models.Recipe, ps pictureSaver) error {
|
||||||
|
@ -254,7 +261,7 @@ func SaveRecipePicture(r *models.Recipe, ps pictureSaver) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetKey gets
|
// GetKey gets
|
||||||
func GetKey(log slog.Logger) error {
|
func GetKey(log *slog.Logger) error {
|
||||||
const op = "parser.GetKey"
|
const op = "parser.GetKey"
|
||||||
|
|
||||||
log.Debug("Updating KEY...")
|
log.Debug("Updating KEY...")
|
||||||
|
@ -267,14 +274,14 @@ func GetKey(log slog.Logger) error {
|
||||||
i := strings.Index(str, "\"key\":")
|
i := strings.Index(str, "\"key\":")
|
||||||
if i != 0 {
|
if i != 0 {
|
||||||
parseKey = str[i+7 : i+47]
|
parseKey = str[i+7 : i+47]
|
||||||
log.Debug("New KEY =", parseKey)
|
log.Debug("New KEY", "key", parseKey)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Errorf("%s: %w", op, KeyNotFoundErr)
|
return fmt.Errorf("%s: %w", op, KeyNotFoundErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetPHPSESSID(log slog.Logger) error {
|
func GetPHPSESSID(log *slog.Logger) error {
|
||||||
const op = "parser.GetPHPSESSID"
|
const op = "parser.GetPHPSESSID"
|
||||||
|
|
||||||
log.Debug("Updating PHPSESSID...")
|
log.Debug("Updating PHPSESSID...")
|
||||||
|
@ -294,7 +301,7 @@ func GetPHPSESSID(log slog.Logger) error {
|
||||||
for _, c := range resp.Response().Cookies() {
|
for _, c := range resp.Response().Cookies() {
|
||||||
if c.Name == "PHPSESSID" {
|
if c.Name == "PHPSESSID" {
|
||||||
PHPSESSID = c.Value
|
PHPSESSID = c.Value
|
||||||
log.Debug("New PHPSESSID =", PHPSESSID)
|
log.Debug("New PHPSESSID", "PHPSESSID", PHPSESSID)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue