diff --git a/internal/lib/stringcv/stringcv.go b/internal/lib/stringcv/stringcv.go new file mode 100644 index 0000000..c675896 --- /dev/null +++ b/internal/lib/stringcv/stringcv.go @@ -0,0 +1,18 @@ +package stringcv + +import ( + "fmt" + "strings" +) + +// GetFilenameFromUrl convert url to filename. +func GetFilenameFromUrl(url string) string { + url_els := strings.Split(url, "/") + return url_els[len(url_els)-1] +} + +// RenameFile changes filename. +func RenameFile(old_filename, new_name string) string { + old_file_els := strings.Split(old_filename, ".") + return fmt.Sprintf("%s.%s", new_name, old_file_els[len(old_file_els)-1]) +} diff --git a/internal/media_storage/minio/minio.go b/internal/media_storage/minio/minio.go index f02e88f..e2f904b 100644 --- a/internal/media_storage/minio/minio.go +++ b/internal/media_storage/minio/minio.go @@ -22,7 +22,7 @@ type ObjStorage struct { } func New(ctx context.Context, addr, user, password string) (*ObjStorage, error) { - const op = "minio.New" + const op = "media_storage.minio.New" minioClient, err := minio.New(addr, &minio.Options{ Creds: credentials.NewStaticV4(user, password, ""), @@ -66,7 +66,7 @@ func New(ctx context.Context, addr, user, password string) (*ObjStorage, error) // Upload file to bucket func (o *ObjStorage) uploadFile(ctx context.Context, bucketName string, objectName string, fileBuffer io.Reader, contentType string, fileSize int64) error { - const op = "minio.UploadFile" + const op = "media_storage.minio.UploadFile" // Upload the zip file with PutObject info, err := o.minio.PutObject(ctx, bucketName, objectName, fileBuffer, fileSize, minio.PutObjectOptions{ContentType: contentType}) if err != nil { @@ -78,7 +78,7 @@ func (o *ObjStorage) uploadFile(ctx context.Context, bucketName string, objectNa // Get file from bucket func (o *ObjStorage) getFile(ctx context.Context, bucketName string, objectName string) (*minio.Object, error) { - const op = "minio.GetFile" + const op = "media_storage.minio.GetFile" // Get object from minio minio_obj, err := o.minio.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{Checksum: true}) if err != nil { @@ -89,7 +89,7 @@ func (o *ObjStorage) getFile(ctx context.Context, bucketName string, objectName // Delete file from bucket func (o *ObjStorage) delFile(ctx context.Context, bucketName string, objectName string) error { - const op = "minio.DelFile" + const op = "media_storage.minio.DelFile" err := o.minio.RemoveObject(ctx, bucketName, objectName, minio.RemoveObjectOptions{ForceDelete: true}) if err != nil { return fmt.Errorf("%s: %w", op, err) @@ -97,6 +97,22 @@ func (o *ObjStorage) delFile(ctx context.Context, bucketName string, objectName return nil } -func (o *ObjStorage) SaveRecipeImage(ctx context.Context) error { - o.uploadFile(ctx, recipeImgBucket) +// SaveRecipeImage saves image to bucket for recipes photos. +func (o *ObjStorage) SaveRecipeImage(ctx context.Context, imageFile io.Reader, filename string, contentType string, fileSize int64) error { + const op = "media_storage.minio.SaveRecipeImage" + err := o.uploadFile(ctx, recipeImgBucket, filename, imageFile, contentType, fileSize) + if err != nil { + return fmt.Errorf("%s: %w", op, err) + } + return err +} + +// RecipeImage gets image from recipe's images bucket by filename. +func (o *ObjStorage) RecipeImage(ctx context.Context, filename string) (*minio.Object, error) { + const op = "media_storage.minio.RecipeImage" + obj, err := o.getFile(ctx, recipeImgBucket, filename) + if err != nil { + return nil, fmt.Errorf("%s: %w", op, err) + } + return obj, err } diff --git a/internal/parser/parser.go b/internal/parser/parser.go index 1df0814..12fd125 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -1,17 +1,21 @@ package parser import ( + "context" "errors" "fmt" "io" "log/slog" + "net/http" "net/url" "recipes/internal/domain/models" + "recipes/internal/lib/stringcv" "strconv" "strings" "sync" "github.com/PuerkitoBio/goquery" + "github.com/google/uuid" "github.com/s32x/httpclient" ) @@ -27,21 +31,34 @@ var ( CookieNotFoundErr = errors.New("cookie not found") NotSuccessReqErr = errors.New("not success request") EmptyLinkErr = errors.New("empty link") + RecipeExistsErr = errors.New("recipe already exists") ) +type pictureSaver interface { + SaveRecipeImage(ctx context.Context, imageFile io.Reader, filename string, contentType string, fileSize int64) error +} + +type recipeSaver interface { + AddRecipe(ctx context.Context, recipe models.Recipe) error +} + +type recipeProvider interface { + RecipeExists(ctx context.Context, title string) (bool, error) +} + // SaveAllPages saves all pages to storage. -func SaveAllPages(log slog.Logger) error { +func SaveAllPages(log slog.Logger, ps pictureSaver, rs recipeSaver, rp recipeProvider) error { const op = "parser.SaveAllPages" // get total log.Debug("Сохраняю страницу 1...") - total, err := SavePage(log, 1) + total, err := SavePage(log, 1, ps, rs, rp) if err != nil { return fmt.Errorf("%s: %w", op, err) } fmt.Println("Total =", total) for i := 2; i <= total; i++ { log.Debug(fmt.Sprintf("Сохраняю страницу %d...\n", i)) - _, err = SavePage(log, i) + _, err = SavePage(log, i, ps, rs, rp) log.Debug(fmt.Sprintf("Страница %d сохранена\n", i)) } @@ -49,7 +66,7 @@ func SaveAllPages(log slog.Logger) error { } // SavePage saves page to storage. -func SavePage(log slog.Logger, page int) (int, error) { +func SavePage(log slog.Logger, page int, ps pictureSaver, rs recipeSaver, rp recipeProvider) (int, error) { const op = "parser.SavePage" var resp GetPageResp @@ -102,20 +119,20 @@ func SavePage(log slog.Logger, page int) (int, error) { var wg sync.WaitGroup wg.Add(len(recipes)) for i := 0; i < len(recipes); i++ { - go func(i int) { + go func(i int, log slog.Logger) { defer wg.Done() - err = recipes[i].GetRecipe() + err = GetRecipe(&recipes[i], ps, rs, rp) if err != nil { - return fmt.Errorf("%s: %w", op, err) + log.Error("Failed to get recipe", "err", fmt.Errorf("%s: %w", op, err)) } - }(i) + }(i, log) } wg.Wait() return resp.Data.Pages, nil } // GetRecipe gets recipe info and saves recipe to storage. -func GetRecipe(r *models.Recipe) error { +func GetRecipe(r *models.Recipe, ps pictureSaver, rs recipeSaver, rp recipeProvider) error { const op = "parser.GetRecipe" if r.Link == "" { @@ -200,65 +217,41 @@ func GetRecipe(r *models.Recipe) error { doc.Find("div.similar-items>a.similar-items__link").Each(func(i int, s *goquery.Selection) { r.Categories = append(r.Categories, s.Text()) }) - // // вывод результатов - // fmt.Println("-------------------") - // fmt.Printf("%+v\n", r) - // fmt.Println("-------------------") // check recipe exists - ex, err := postgres.DB.RecipeExists(r.Title) // interface! + ex, err := rp.RecipeExists(context.Background(), r.Title) // interface! if err != nil || ex { - return fmt.Errorf("%s: %w", op, fmt.Errorf("recipe already exists")) + return fmt.Errorf("%s: %w", op, RecipeExistsErr) } // save picture - err = r.SaveRecipePicture() - // add to database - var final_recipe models.Recipe = models.Recipe{ - Title: r.Title, - Description: r.Description, - Image: r.Image, - CookingTime: r.CookingTime, - Link: r.Link, - ServingsNum: r.ServingsNum, - Calories: r.Calories, - Ingredients: r.Ingredients, - Recipe_steps: r.Recipe_steps, - Advices: r.Advices, - Categories: r.Categories, + err = SaveRecipePicture(r, ps) + if err != nil { + return err } // insert recipe - err = postgres.DB.AddRecipe(final_recipe) + err = rs.AddRecipe(context.Background(), *r) return fmt.Errorf("%s: %w", op, err) } -// func (r *Recipe) SaveRecipePicture() error { -// resp, err := http.Get(r.Image) -// if err != nil { +func SaveRecipePicture(r *models.Recipe, ps pictureSaver) error { + const op = "parser.SaveRecipePicture" -// return err -// } -// defer resp.Body.Close() -// content_len, _ := strconv.ParseInt(resp.Header["Content-Length"][0], 10, 64) -// // change name to generated uuid -// filename := renamefile(getFilenameFromUrl(r.Image), uuid.NewString()) -// // upload to minio -// err = cminio.UploadFile(cminio.RecipeImg, filename, resp.Body, resp.Header["Content-Type"][0], content_len) - -// // change to filename -// r.Image = filename -// return err -// } - -// // url to filename -// func getFilenameFromUrl(url string) string { -// url_els := strings.Split(url, "/") -// return url_els[len(url_els)-1] -// } - -// // change file name -// func renamefile(old_filename, new_name string) string { -// old_file_els := strings.Split(old_filename, ".") -// return fmt.Sprintf("%s.%s", new_name, old_file_els[len(old_file_els)-1]) -// } + resp, err := http.Get(r.Image) + if err != nil { + return fmt.Errorf("%s: %w", op, err) + } + defer resp.Body.Close() + content_len, _ := strconv.ParseInt(resp.Header["Content-Length"][0], 10, 64) + // change name to generated uuid + filename := stringcv.RenameFile(stringcv.GetFilenameFromUrl(r.Image), uuid.NewString()) + // upload to storage + err = ps.SaveRecipeImage(context.Background(), resp.Body, filename, resp.Header["Content-Type"][0], content_len) + if err != nil { + return err + } + // change to filename + r.Image = filename + return err +} // GetKey gets func GetKey(log slog.Logger) error {