From f363ad2f2710e36bfaa6dd932e9ddbe38463edb7 Mon Sep 17 00:00:00 2001 From: yash Date: Sat, 20 Jan 2024 21:33:49 +0300 Subject: [PATCH] add minio --- go.mod | 8 + go.sum | 47 ++++ internal/media_storage/minio/minio.go | 90 ++++++- internal/parser/parser.go | 306 ++++++++++++++++++++++ internal/parser/structs.go | 14 + internal/storage/postgresql/postgresql.go | 4 +- 6 files changed, 464 insertions(+), 5 deletions(-) create mode 100644 internal/parser/structs.go diff --git a/go.mod b/go.mod index a4e192b..5ca3047 100644 --- a/go.mod +++ b/go.mod @@ -13,11 +13,15 @@ require ( require ( github.com/BurntSushi/toml v1.2.1 // indirect + github.com/PuerkitoBio/goquery v1.8.1 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.1.2 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/charmbracelet/lipgloss v0.9.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/google/uuid v1.5.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -35,7 +39,9 @@ require ( github.com/mattn/go-isatty v0.0.18 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go v6.0.14+incompatible // indirect github.com/minio/sha256-simd v1.0.1 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/muesli/reflow v0.3.0 // indirect @@ -43,6 +49,7 @@ require ( github.com/rivo/uniseg v0.2.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/rs/xid v1.5.0 // indirect + github.com/s32x/httpclient v0.0.0-20220217184346-6df4d4d51c14 // indirect github.com/sirupsen/logrus v1.9.3 // indirect go.uber.org/atomic v1.7.0 // indirect golang.org/x/crypto v0.17.0 // indirect @@ -53,5 +60,6 @@ require ( golang.org/x/text v0.14.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + h12.io/socks v1.0.3 // indirect olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect ) diff --git a/go.sum b/go.sum index c14e2c8..74d1d26 100644 --- a/go.sum +++ b/go.sum @@ -4,12 +4,18 @@ github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= +github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v4 v4.1.2 h1:6Yo7N8UP2K6LWZnW94DLVSSrbobcWdVzAYOisuDPIFo= +github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= @@ -34,6 +40,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -43,6 +51,7 @@ github.com/golang-migrate/migrate/v4 v4.17.0/go.mod h1:+Cp2mtLP4/aXDTKb9wmXYitdr github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/h12w/go-socks5 v0.0.0-20200522160539-76189e178364/go.mod h1:eDJQioIyy4Yn3MVivT7rv/39gAJTrA7lgmYr8EW950c= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -82,10 +91,14 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= +github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw= github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -103,6 +116,7 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -116,6 +130,8 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/s32x/httpclient v0.0.0-20220217184346-6df4d4d51c14 h1:7X+d8s674m+oOrb2YbbBrtA1Ey1puN9mBvHp2SWfkig= +github.com/s32x/httpclient v0.0.0-20220217184346-6df4d4d51c14/go.mod h1:FqlhGa3u28mSvrE2aDhpKc1h1WpEcjmrXFkN/e2qnxQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -123,27 +139,56 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -152,5 +197,7 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +h12.io/socks v1.0.3 h1:Ka3qaQewws4j4/eDQnOdpr4wXsC//dXtWvftlIcCQUo= +h12.io/socks v1.0.3/go.mod h1:AIhxy1jOId/XCz9BO+EIgNL2rQiPTBNnOfnVnQ+3Eck= olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 h1:slmdOY3vp8a7KQbHkL+FLbvbkgMqmXojpFUO/jENuqQ= olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3/go.mod h1:oVgVk4OWVDi43qWBEyGhXgYxt7+ED4iYNpTngSLX2Iw= diff --git a/internal/media_storage/minio/minio.go b/internal/media_storage/minio/minio.go index 02dabd5..9f74643 100644 --- a/internal/media_storage/minio/minio.go +++ b/internal/media_storage/minio/minio.go @@ -1,14 +1,98 @@ package minio import ( + "context" + "fmt" + "io" + "log" + "github.com/minio/minio-go/v7" - _ "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) +// Bucket names +var ( + RecipeImg string = "recipeimg" +) + +const location string = "eu_ru" + type ObjStorage struct { minio *minio.Client } -func New() *ObjStorage { - return &ObjStorage{} +func New(ctx context.Context, addr, user, password string) (*ObjStorage, error) { + const op = "minio.New" + + minioClient, err := minio.New(addr, &minio.Options{ + Creds: credentials.NewStaticV4(user, password, ""), + Secure: false, + }) + if err != nil { + return nil, fmt.Errorf("%s: %w", op, err) + } + + return &ObjStorage{minio: minioClient}, nil +} + +// // MinioConnection func for opening minio connection. +// func MinioConnection(bucketName string) (*minio.Client, error) { +// ctx := context.Background() +// useSSL := false +// // Initialize minio client object. +// minioClient, errInit := minio.New(config.Conf.MINIO_ADDR, &minio.Options{ +// Creds: credentials.NewStaticV4(config.Conf.MINIO_ROOT_USER, config.Conf.MINIO_ROOT_PASSWORD, ""), +// Secure: useSSL, +// }) +// if errInit != nil { +// return nil, errInit +// } +// // Check exists +// exists, errBucketExists := minioClient.BucketExists(ctx, bucketName) +// if errBucketExists != nil { +// return nil, errBucketExists +// } +// if !exists { +// // Create bucket +// err := minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: location}) +// if err != nil { +// return nil, err +// } else { +// log.Printf("Successfully created %s\n", bucketName) +// } +// } +// return minioClient, errInit +// } + +// Upload file to bucket +func (o *ObjStorage) UploadFile(ctx context.Context, bucketName string, objectName string, fileBuffer io.Reader, contentType string, fileSize int64) error { + const op = "minio.UploadFile" + // Upload the zip file with PutObject + info, err := o.minio.PutObject(ctx, bucketName, objectName, fileBuffer, fileSize, minio.PutObjectOptions{ContentType: contentType}) + if err != nil { + return fmt.Errorf("%s: %w", op, err) + } + log.Printf("Successfully uploaded %s of size %d\n", objectName, info.Size) + return nil +} + +// Get file from bucket +func (o *ObjStorage) GetFile(ctx context.Context, bucketName string, objectName string) (*minio.Object, error) { + const op = "minio.GetFile" + // Get object from minio + minio_obj, err := o.minio.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{Checksum: true}) + if err != nil { + return nil, fmt.Errorf("%s: %w", op, err) + } + return minio_obj, nil +} + +// Delete file from bucket +func (o *ObjStorage) DelFile(ctx context.Context, bucketName string, objectName string) error { + const op = "minio.DelFile" + err := o.minio.RemoveObject(ctx, bucketName, objectName, minio.RemoveObjectOptions{ForceDelete: true}) + if err != nil { + return fmt.Errorf("%s: %w", op, err) + } + return nil } diff --git a/internal/parser/parser.go b/internal/parser/parser.go index 0bfe2c2..635d38f 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -1 +1,307 @@ package parser + +import ( + "errors" + "fmt" + "io" + "log/slog" + "net/url" + "recipes/internal/domain/models" + "strconv" + "strings" + "sync" + + "github.com/PuerkitoBio/goquery" + "github.com/s32x/httpclient" +) + +const baseUrl string = "https://www.vsegdavkusno.ru" + +var client *httpclient.Client = httpclient.New().WithBaseURL(baseUrl) + +var PHPSESSID string +var parseKey string + +// SaveAllPages saves all pages to storage. +func SaveAllPages(log slog.Logger) error { + const op = "parser.SaveAllPages" + // get total + log.Debug("Сохраняю страницу 1...") + total, err := SavePage(log, 1) + if err != nil { + return fmt.Errorf("%s: %w", op, err) + } + fmt.Println("Total =", total) + for i := 2; i <= total; i++ { + log.Debug(fmt.Sprintf("Сохраняю страницу %d...\n", i)) + _, err = SavePage(log, i) + + log.Debug(fmt.Sprintf("Страница %d сохранена\n", i)) + } + return nil +} + +// SavePage saves page to storage. +func SavePage(log slog.Logger, page int) (int, error) { + const op = "parser.SavePage" + + var resp GetPageResp + var body io.Reader + for i := 0; i <= 3; i++ { + // make form + form := make(url.Values) + form.Add("page", fmt.Sprint(page)) + form.Add("action", "filter") + form.Add("pageId", "7") + form.Add("key", parseKey) + // send request + err := client. + WithHeader("Cookie", fmt.Sprintf("PHPSESSID=%s", PHPSESSID)). + WithHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"). + Post("/assets/components/msearch2/action.php").WithForm(form).JSON(&resp) + if err != nil { + return 0, fmt.Errorf("%s: %w", op, err) + } + // update PHPSESSID and key + if resp.Message == "Could not load config" { + GetPHPSESSID(log) + GetKey(log) + continue + } + if !resp.Success { + err = errors.New("not success request") + return 0, fmt.Errorf("%s: %w", op, err) + } + body = strings.NewReader(resp.Data.Results) + break + } + doc, err := goquery.NewDocumentFromReader(body) + if err != nil { + return 0, fmt.Errorf("%s: %w", op, err) + } + var recipes []models.Recipe + doc.Find("div.recipe-card").Each(func(i int, s *goquery.Selection) { + var recipe models.Recipe + recipe.Title = s.Find("div.recipe-card__title").Text() + // recipe.Time = strings.ReplaceAll(strings.ReplaceAll(s.Find("div.recipe-card__time").Text(), "\n", ""), " ", "") + recipe.Image, _ = s.Find("img").Attr("src") + recipe.Image = strings.Replace(recipe.Image, "image_366", "image_732", 1) + recipe.Image = fmt.Sprintf("%s%s", baseUrl, recipe.Image) + recipe.Link, _ = s.Find("a.recipe-card__link").Attr("href") + recipe.Link = fmt.Sprintf("/%s", recipe.Link) + + recipes = append(recipes, recipe) + }) + + var wg sync.WaitGroup + wg.Add(len(recipes)) + for i := 0; i < len(recipes); i++ { + go func(i int) { + defer wg.Done() + err = recipes[i].GetRecipe() + if err != nil { + return fmt.Errorf("%s: %w", op, err) + } + }(i) + } + wg.Wait() + return resp.Data.Pages, nil +} + +// GetRecipe gets recipe info and saves recipe to storage. +func GetRecipe(r *models.Recipe) error { + const op = "parser.GetRecipe" + + if r.Link == "" { + return fmt.Errorf("%s: %w", op, errors.New("empty link")) + } + // send request + body, err := client. + WithHeader("Cookie", fmt.Sprintf("PHPSESSID=%s", PHPSESSID)). + Post(r.Link).String() + if err != nil { + return fmt.Errorf("%s: %w", op, err) + } + bodyr := strings.NewReader(body) + doc, err := goquery.NewDocumentFromReader(bodyr) + if err != nil { + return fmt.Errorf("%s: %w", op, err) + } + // описание + r.Description = strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(doc.Find("div.card-binfo__description").Text(), " ", ""), "\n", ""), "\t", "") + // Время готовки, кол-во порций + doc.Find("div.recipe-summary-items>div.recipe-summary-item").Each(func(i int, s *goquery.Selection) { + label := s.Find("div.recipe-summary-item__label").Text() + value := strings.ReplaceAll(s.Find("div.recipe-summary-item__value").Text(), "\n", "") + switch label { + case "Время приготовления:": + r.CookingTime = value + case "Количество порций:": + count, _ := strconv.Atoi(value) + r.ServingsNum = uint(count) + case "Калорийность:": + r.Calories = value + } + }) + doc.Find("div.ingredients").Each(func(i int, s *goquery.Selection) { + // ингридиенты для рецепта + var ingredients models.RecipeIngredients + ingredients.Title = s.Find("div.ingredients__title").Text() + s.Find("ul.ingredients__list>li").Each(func(i int, q *goquery.Selection) { + ingredient := strings.ReplaceAll(strings.ReplaceAll(q.Text(), " ", ""), "\n", "") + ingredients.Ingredients = append(ingredients.Ingredients, ingredient) + }) + if len(ingredients.Ingredients) != 0 { + r.Ingredients = append(r.Ingredients, ingredients) + } + // шаги рецепта + var recipe_steps []string + s.Find("div.recipe-rich>ol>li>span").Each(func(i int, q *goquery.Selection) { + recipe_step := strings.ReplaceAll(strings.ReplaceAll(q.Text(), " ", ""), "\n", "") + recipe_steps = append(recipe_steps, recipe_step) + }) + if len(recipe_steps) != 0 { + r.Recipe_steps = recipe_steps + } + // рекомендации + var advices []string + s.Find("div.recipe-footer__additional-text>p").Each(func(i int, q *goquery.Selection) { + if q.Find("br").Length() > 0 { + html, err := q.Html() + if err != nil { + + return fmt.Errorf("%s: %w", op, err) + } + advice_list := q.SetHtml(strings.Replace(html, "
", "\n", -1)).Text() + advices_arr := strings.Split(advice_list, "\n") + var advices_arr_res []string = make([]string, 0, len(advices_arr)) + for _, a := range advices_arr { + if a != "" { + advices_arr_res = append(advices_arr_res, a) + } + } + advices = append(advices, advices_arr_res...) + } else { + var advice string = q.Text() + advices = append(advices, advice) + } + }) + if len(advices) != 0 { + // fmt.Printf("LEN ADVICES = %d", len(advices)) + r.Advices = advices + } + }) + // категории + doc.Find("div.similar-items>a.similar-items__link").Each(func(i int, s *goquery.Selection) { + r.Categories = append(r.Categories, s.Text()) + }) + // // вывод результатов + // fmt.Println("-------------------") + // fmt.Printf("%+v\n", r) + // fmt.Println("-------------------") + // check recipe exists + ex, err := postgres.DB.RecipeExists(r.Title) + if err != nil || ex { + return fmt.Errorf("%s: %w", op, fmt.Errorf("recipe already exists")) + } + // save picture + err = r.SaveRecipePicture() + // add to database + var final_recipe models.Recipe = models.Recipe{ + Title: r.Title, + Description: r.Description, + Image: r.Image, + CookingTime: r.CookingTime, + Link: r.Link, + ServingsNum: r.ServingsNum, + Calories: r.Calories, + Ingredients: r.Ingredients, + Recipe_steps: r.Recipe_steps, + Advices: r.Advices, + Categories: r.Categories, + } + // insert recipe + err = postgres.DB.AddRecipe(final_recipe) + return fmt.Errorf("%s: %w", op, err) +} + +// func (r *Recipe) SaveRecipePicture() error { +// resp, err := http.Get(r.Image) +// if err != nil { + +// return err +// } +// defer resp.Body.Close() +// content_len, _ := strconv.ParseInt(resp.Header["Content-Length"][0], 10, 64) +// // change name to generated uuid +// filename := renamefile(getFilenameFromUrl(r.Image), uuid.NewString()) +// // upload to minio +// err = cminio.UploadFile(cminio.RecipeImg, filename, resp.Body, resp.Header["Content-Type"][0], content_len) + +// // change to filename +// r.Image = filename +// return err +// } + +// // url to filename +// func getFilenameFromUrl(url string) string { +// url_els := strings.Split(url, "/") +// return url_els[len(url_els)-1] +// } + +// // change file name +// func renamefile(old_filename, new_name string) string { +// old_file_els := strings.Split(old_filename, ".") +// return fmt.Sprintf("%s.%s", new_name, old_file_els[len(old_file_els)-1]) +// } + +// GetKey gets +func GetKey(log slog.Logger) error { + const op = "parser.GetKey" + + log.Debug("Updating KEY...") + defer log.Debug("KEY updated") + str, err := client.WithHeader("Cookie", fmt.Sprintf("PHPSESSID=%s", PHPSESSID)).Post("/recipes").String() + if err != nil { + + return fmt.Errorf("%s: %w", op, err) + } + i := strings.Index(str, "\"key\":") + if i != 0 { + parseKey = str[i+7 : i+47] + log.Debug("New KEY =", parseKey) + return nil + } + err = errors.New("key not found") + + return fmt.Errorf("%s: %w", op, err) +} + +func GetPHPSESSID(log slog.Logger) error { + const op = "parser.GetPHPSESSID" + + log.Debug("Updating PHPSESSID...") + defer log.Debug("PHPSESSID updated") + form := make(url.Values) + form.Add("page", "1") + form.Add("action", "filter") + form.Add("pageId", "7") + form.Add("key", "-") + resp, err := client. + WithHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"). + WithHeader("Cookie", ""). + Post("/assets/components/msearch2/action.php").WithForm(form).Do() + if err != nil { + return fmt.Errorf("%s: %w", op, err) + } + for _, c := range resp.Response().Cookies() { + if c.Name == "PHPSESSID" { + PHPSESSID = c.Value + log.Debug("New PHPSESSID =", PHPSESSID) + return nil + } + } + err = errors.New("cookie not found") + + return fmt.Errorf("%s: %w", op, err) +} diff --git a/internal/parser/structs.go b/internal/parser/structs.go new file mode 100644 index 0000000..8253c60 --- /dev/null +++ b/internal/parser/structs.go @@ -0,0 +1,14 @@ +package parser + +type GetPageResp struct { + Success bool `json:"success"` + Message string `json:"message"` + Data struct { + Results string `json:"results"` + Pagination string `json:"pagination"` + Total int `json:"total"` + Page int `json:"page"` + Pages int `json:"pages"` + Log string `json:"log"` + } `json:"data"` +} diff --git a/internal/storage/postgresql/postgresql.go b/internal/storage/postgresql/postgresql.go index 2de69bc..6aeefc4 100644 --- a/internal/storage/postgresql/postgresql.go +++ b/internal/storage/postgresql/postgresql.go @@ -12,10 +12,10 @@ type Storage struct { db *pgxpool.Pool } -func New(user, password, addr, dbname string) (*Storage, error) { +func New(ctx context.Context, user, password, addr, dbname string) (*Storage, error) { const op = "storage.postgresql.New" - pool, err := pgxpool.New(context.Background(), fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", user, password, addr, dbname)) + pool, err := pgxpool.New(ctx, fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", user, password, addr, dbname)) if err != nil { return nil, fmt.Errorf("%s: %w", op, err) }