yash
/
recipes2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

minio save/get recipe; parser fixings

This commit is contained in:
yash 2024-01-21 13:10:06 +03:00
parent 67e913ff3a
commit d7c777a2ca
3 changed files with 91 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
internal/lib/stringcv/stringcv.go Normal file
View File

@ -0,0 +1,18 @@
package stringcv
import (
"fmt"
"strings"
)
// GetFilenameFromUrl convert url to filename.
func GetFilenameFromUrl(url string) string {
url_els := strings.Split(url, "/")
return url_els[len(url_els)-1]
}
// RenameFile changes filename.
func RenameFile(old_filename, new_name string) string {
old_file_els := strings.Split(old_filename, ".")
return fmt.Sprintf("%s.%s", new_name, old_file_els[len(old_file_els)-1])
}

28
internal/media_storage/minio/minio.go
View File

@ -22,7 +22,7 @@ type ObjStorage struct {
}
func New(ctx context.Context, addr, user, password string) (*ObjStorage, error) {
const op = "minio.New"
const op = "media_storage.minio.New"
minioClient, err := minio.New(addr, &minio.Options{
Creds: credentials.NewStaticV4(user, password, ""),
@ -66,7 +66,7 @@ func New(ctx context.Context, addr, user, password string) (*ObjStorage, error)
// Upload file to bucket
func (o *ObjStorage) uploadFile(ctx context.Context, bucketName string, objectName string, fileBuffer io.Reader, contentType string, fileSize int64) error {
const op = "minio.UploadFile"
const op = "media_storage.minio.UploadFile"
// Upload the zip file with PutObject
info, err := o.minio.PutObject(ctx, bucketName, objectName, fileBuffer, fileSize, minio.PutObjectOptions{ContentType: contentType})
if err != nil {
@ -78,7 +78,7 @@ func (o *ObjStorage) uploadFile(ctx context.Context, bucketName string, objectNa
// Get file from bucket
func (o *ObjStorage) getFile(ctx context.Context, bucketName string, objectName string) (*minio.Object, error) {
const op = "minio.GetFile"
const op = "media_storage.minio.GetFile"
// Get object from minio
minio_obj, err := o.minio.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{Checksum: true})
if err != nil {
@ -89,7 +89,7 @@ func (o *ObjStorage) getFile(ctx context.Context, bucketName string, objectName
// Delete file from bucket
func (o *ObjStorage) delFile(ctx context.Context, bucketName string, objectName string) error {
const op = "minio.DelFile"
const op = "media_storage.minio.DelFile"
err := o.minio.RemoveObject(ctx, bucketName, objectName, minio.RemoveObjectOptions{ForceDelete: true})
if err != nil {
return fmt.Errorf("%s: %w", op, err)
@ -97,6 +97,22 @@ func (o *ObjStorage) delFile(ctx context.Context, bucketName string, objectName
return nil
}
func (o *ObjStorage) SaveRecipeImage(ctx context.Context) error {
o.uploadFile(ctx, recipeImgBucket)
// SaveRecipeImage saves image to bucket for recipes photos.
func (o *ObjStorage) SaveRecipeImage(ctx context.Context, imageFile io.Reader, filename string, contentType string, fileSize int64) error {
const op = "media_storage.minio.SaveRecipeImage"
err := o.uploadFile(ctx, recipeImgBucket, filename, imageFile, contentType, fileSize)
if err != nil {
return fmt.Errorf("%s: %w", op, err)
}
return err
}
// RecipeImage gets image from recipe's images bucket by filename.
func (o *ObjStorage) RecipeImage(ctx context.Context, filename string) (*minio.Object, error) {
const op = "media_storage.minio.RecipeImage"
obj, err := o.getFile(ctx, recipeImgBucket, filename)
if err != nil {
return nil, fmt.Errorf("%s: %w", op, err)
}
return obj, err
}

109
internal/parser/parser.go
View File

@ -1,17 +1,21 @@
package parser
import (
"context"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"recipes/internal/domain/models"
"recipes/internal/lib/stringcv"
"strconv"
"strings"
"sync"
"github.com/PuerkitoBio/goquery"
"github.com/google/uuid"
"github.com/s32x/httpclient"
)
@ -27,21 +31,34 @@ var (
CookieNotFoundErr = errors.New("cookie not found")
NotSuccessReqErr = errors.New("not success request")
EmptyLinkErr = errors.New("empty link")
RecipeExistsErr = errors.New("recipe already exists")
)
type pictureSaver interface {
SaveRecipeImage(ctx context.Context, imageFile io.Reader, filename string, contentType string, fileSize int64) error
}
type recipeSaver interface {
AddRecipe(ctx context.Context, recipe models.Recipe) error
}
type recipeProvider interface {
RecipeExists(ctx context.Context, title string) (bool, error)
}
// SaveAllPages saves all pages to storage.
func SaveAllPages(log slog.Logger) error {
func SaveAllPages(log slog.Logger, ps pictureSaver, rs recipeSaver, rp recipeProvider) error {
const op = "parser.SaveAllPages"
// get total
log.Debug("Сохраняю страницу 1...")
total, err := SavePage(log, 1)
total, err := SavePage(log, 1, ps, rs, rp)
if err != nil {
return fmt.Errorf("%s: %w", op, err)
}
fmt.Println("Total =", total)
for i := 2; i <= total; i++ {
log.Debug(fmt.Sprintf("Сохраняю страницу %d...\n", i))
_, err = SavePage(log, i)
_, err = SavePage(log, i, ps, rs, rp)
log.Debug(fmt.Sprintf("Страница %d сохранена\n", i))
}
@ -49,7 +66,7 @@ func SaveAllPages(log slog.Logger) error {
}
// SavePage saves page to storage.
func SavePage(log slog.Logger, page int) (int, error) {
func SavePage(log slog.Logger, page int, ps pictureSaver, rs recipeSaver, rp recipeProvider) (int, error) {
const op = "parser.SavePage"
var resp GetPageResp
@ -102,20 +119,20 @@ func SavePage(log slog.Logger, page int) (int, error) {
var wg sync.WaitGroup
wg.Add(len(recipes))
for i := 0; i < len(recipes); i++ {
go func(i int) {
go func(i int, log slog.Logger) {
defer wg.Done()
err = recipes[i].GetRecipe()
err = GetRecipe(&recipes[i], ps, rs, rp)
if err != nil {
return fmt.Errorf("%s: %w", op, err)
log.Error("Failed to get recipe", "err", fmt.Errorf("%s: %w", op, err))
}
}(i)
}(i, log)
}
wg.Wait()
return resp.Data.Pages, nil
}
// GetRecipe gets recipe info and saves recipe to storage.
func GetRecipe(r *models.Recipe) error {
func GetRecipe(r *models.Recipe, ps pictureSaver, rs recipeSaver, rp recipeProvider) error {
const op = "parser.GetRecipe"
if r.Link == "" {
@ -200,65 +217,41 @@ func GetRecipe(r *models.Recipe) error {
doc.Find("div.similar-items>a.similar-items__link").Each(func(i int, s *goquery.Selection) {
r.Categories = append(r.Categories, s.Text())
})
// // вывод результатов
// fmt.Println("-------------------")
// fmt.Printf("%+v\n", r)
// fmt.Println("-------------------")
// check recipe exists
ex, err := postgres.DB.RecipeExists(r.Title) // interface!
ex, err := rp.RecipeExists(context.Background(), r.Title) // interface!
if err != nil || ex {
return fmt.Errorf("%s: %w", op, fmt.Errorf("recipe already exists"))
return fmt.Errorf("%s: %w", op, RecipeExistsErr)
}
// save picture
err = r.SaveRecipePicture()
// add to database
var final_recipe models.Recipe = models.Recipe{
Title: r.Title,
Description: r.Description,
Image: r.Image,
CookingTime: r.CookingTime,
Link: r.Link,
ServingsNum: r.ServingsNum,
Calories: r.Calories,
Ingredients: r.Ingredients,
Recipe_steps: r.Recipe_steps,
Advices: r.Advices,
Categories: r.Categories,
err = SaveRecipePicture(r, ps)
if err != nil {
return err
}
// insert recipe
err = postgres.DB.AddRecipe(final_recipe)
err = rs.AddRecipe(context.Background(), *r)
return fmt.Errorf("%s: %w", op, err)
}
// func (r *Recipe) SaveRecipePicture() error {
// resp, err := http.Get(r.Image)
// if err != nil {
func SaveRecipePicture(r *models.Recipe, ps pictureSaver) error {
const op = "parser.SaveRecipePicture"
// return err
// }
// defer resp.Body.Close()
// content_len, _ := strconv.ParseInt(resp.Header["Content-Length"][0], 10, 64)
// // change name to generated uuid
// filename := renamefile(getFilenameFromUrl(r.Image), uuid.NewString())
// // upload to minio
// err = cminio.UploadFile(cminio.RecipeImg, filename, resp.Body, resp.Header["Content-Type"][0], content_len)
// // change to filename
// r.Image = filename
// return err
// }
// // url to filename
// func getFilenameFromUrl(url string) string {
// url_els := strings.Split(url, "/")
// return url_els[len(url_els)-1]
// }
// // change file name
// func renamefile(old_filename, new_name string) string {
// old_file_els := strings.Split(old_filename, ".")
// return fmt.Sprintf("%s.%s", new_name, old_file_els[len(old_file_els)-1])
// }
resp, err := http.Get(r.Image)
if err != nil {
return fmt.Errorf("%s: %w", op, err)
}
defer resp.Body.Close()
content_len, _ := strconv.ParseInt(resp.Header["Content-Length"][0], 10, 64)
// change name to generated uuid
filename := stringcv.RenameFile(stringcv.GetFilenameFromUrl(r.Image), uuid.NewString())
// upload to storage
err = ps.SaveRecipeImage(context.Background(), resp.Body, filename, resp.Header["Content-Type"][0], content_len)
if err != nil {
return err
}
// change to filename
r.Image = filename
return err
}
// GetKey gets
func GetKey(log slog.Logger) error {