add minio

go.mod

@@ -13,11 +13,15 @@ require (
 
 require (
 	github.com/BurntSushi/toml v1.2.1 // indirect
+	github.com/PuerkitoBio/goquery v1.8.1 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.1.2 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/charmbracelet/lipgloss v0.9.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/go-ini/ini v1.67.0 // indirect
 	github.com/go-logfmt/logfmt v0.6.0 // indirect
 	github.com/google/uuid v1.5.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -35,7 +39,9 @@ require (
 	github.com/mattn/go-isatty v0.0.18 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/minio-go v6.0.14+incompatible // indirect
 	github.com/minio/sha256-simd v1.0.1 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/muesli/reflow v0.3.0 // indirect
@@ -43,6 +49,7 @@ require (
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/rs/xid v1.5.0 // indirect
+	github.com/s32x/httpclient v0.0.0-20220217184346-6df4d4d51c14 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	golang.org/x/crypto v0.17.0 // indirect
@@ -53,5 +60,6 @@ require (
 	golang.org/x/text v0.14.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	h12.io/socks v1.0.3 // indirect
 	olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect
 )

go.sum

@@ -4,12 +4,18 @@ github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
+github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
+github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
 github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
+github.com/cenkalti/backoff/v4 v4.1.2 h1:6Yo7N8UP2K6LWZnW94DLVSSrbobcWdVzAYOisuDPIFo=
+github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg=
@@ -34,6 +40,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
+github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
 github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -43,6 +51,7 @@ github.com/golang-migrate/migrate/v4 v4.17.0/go.mod h1:+Cp2mtLP4/aXDTKb9wmXYitdr
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
 github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/h12w/go-socks5 v0.0.0-20200522160539-76189e178364/go.mod h1:eDJQioIyy4Yn3MVivT7rv/39gAJTrA7lgmYr8EW950c=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -82,10 +91,14 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o=
+github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8=
 github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw=
 github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs=
 github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
 github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
 github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -103,6 +116,7 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
 github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -116,6 +130,8 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/s32x/httpclient v0.0.0-20220217184346-6df4d4d51c14 h1:7X+d8s674m+oOrb2YbbBrtA1Ey1puN9mBvHp2SWfkig=
+github.com/s32x/httpclient v0.0.0-20220217184346-6df4d4d51c14/go.mod h1:FqlhGa3u28mSvrE2aDhpKc1h1WpEcjmrXFkN/e2qnxQ=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -123,27 +139,56 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
 golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
 golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
 golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
 golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
 golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
@@ -152,5 +197,7 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+h12.io/socks v1.0.3 h1:Ka3qaQewws4j4/eDQnOdpr4wXsC//dXtWvftlIcCQUo=
+h12.io/socks v1.0.3/go.mod h1:AIhxy1jOId/XCz9BO+EIgNL2rQiPTBNnOfnVnQ+3Eck=
 olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 h1:slmdOY3vp8a7KQbHkL+FLbvbkgMqmXojpFUO/jENuqQ=
 olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3/go.mod h1:oVgVk4OWVDi43qWBEyGhXgYxt7+ED4iYNpTngSLX2Iw=

internal/media_storage/minio/minio.go

@@ -1,14 +1,98 @@
 package minio
 
 import (
+	"context"
+	"fmt"
+	"io"
+	"log"
+
 	"github.com/minio/minio-go/v7"
-	_ "github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
 )
 
+// Bucket names
+var (
+	RecipeImg string = "recipeimg"
+)
+
+const location string = "eu_ru"
+
 type ObjStorage struct {
 	minio *minio.Client
 }
 
-func New() *ObjStorage {
+func New(ctx context.Context, addr, user, password string) (*ObjStorage, error) {
-	return &ObjStorage{}
+	const op = "minio.New"
+
+	minioClient, err := minio.New(addr, &minio.Options{
+		Creds:  credentials.NewStaticV4(user, password, ""),
+		Secure: false,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", op, err)
+	}
+
+	return &ObjStorage{minio: minioClient}, nil
+}
+
+// // MinioConnection func for opening minio connection.
+// func MinioConnection(bucketName string) (*minio.Client, error) {
+// 	ctx := context.Background()
+// 	useSSL := false
+// 	// Initialize minio client object.
+// 	minioClient, errInit := minio.New(config.Conf.MINIO_ADDR, &minio.Options{
+// 		Creds:  credentials.NewStaticV4(config.Conf.MINIO_ROOT_USER, config.Conf.MINIO_ROOT_PASSWORD, ""),
+// 		Secure: useSSL,
+// 	})
+// 	if errInit != nil {
+// 		return nil, errInit
+// 	}
+// 	// Check exists
+// 	exists, errBucketExists := minioClient.BucketExists(ctx, bucketName)
+// 	if errBucketExists != nil {
+// 		return nil, errBucketExists
+// 	}
+// 	if !exists {
+// 		// Create bucket
+// 		err := minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: location})
+// 		if err != nil {
+// 			return nil, err
+// 		} else {
+// 			log.Printf("Successfully created %s\n", bucketName)
+// 		}
+// 	}
+// 	return minioClient, errInit
+// }
+
+// Upload file to bucket
+func (o *ObjStorage) UploadFile(ctx context.Context, bucketName string, objectName string, fileBuffer io.Reader, contentType string, fileSize int64) error {
+	const op = "minio.UploadFile"
+	// Upload the zip file with PutObject
+	info, err := o.minio.PutObject(ctx, bucketName, objectName, fileBuffer, fileSize, minio.PutObjectOptions{ContentType: contentType})
+	if err != nil {
+		return fmt.Errorf("%s: %w", op, err)
+	}
+	log.Printf("Successfully uploaded %s of size %d\n", objectName, info.Size)
+	return nil
+}
+
+// Get file from bucket
+func (o *ObjStorage) GetFile(ctx context.Context, bucketName string, objectName string) (*minio.Object, error) {
+	const op = "minio.GetFile"
+	// Get object from minio
+	minio_obj, err := o.minio.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{Checksum: true})
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", op, err)
+	}
+	return minio_obj, nil
+}
+
+// Delete file from bucket
+func (o *ObjStorage) DelFile(ctx context.Context, bucketName string, objectName string) error {
+	const op = "minio.DelFile"
+	err := o.minio.RemoveObject(ctx, bucketName, objectName, minio.RemoveObjectOptions{ForceDelete: true})
+	if err != nil {
+		return fmt.Errorf("%s: %w", op, err)
+	}
+	return nil
 }

internal/parser/parser.go

@@ -1 +1,307 @@
 package parser
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/url"
+	"recipes/internal/domain/models"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/s32x/httpclient"
+)
+
+const baseUrl string = "https://www.vsegdavkusno.ru"
+
+var client *httpclient.Client = httpclient.New().WithBaseURL(baseUrl)
+
+var PHPSESSID string
+var parseKey string
+
+// SaveAllPages saves all pages to storage.
+func SaveAllPages(log slog.Logger) error {
+	const op = "parser.SaveAllPages"
+	// get total
+	log.Debug("Сохраняю страницу 1...")
+	total, err := SavePage(log, 1)
+	if err != nil {
+		return fmt.Errorf("%s: %w", op, err)
+	}
+	fmt.Println("Total =", total)
+	for i := 2; i <= total; i++ {
+		log.Debug(fmt.Sprintf("Сохраняю страницу %d...\n", i))
+		_, err = SavePage(log, i)
+
+		log.Debug(fmt.Sprintf("Страница %d сохранена\n", i))
+	}
+	return nil
+}
+
+// SavePage saves page to storage.
+func SavePage(log slog.Logger, page int) (int, error) {
+	const op = "parser.SavePage"
+
+	var resp GetPageResp
+	var body io.Reader
+	for i := 0; i <= 3; i++ {
+		// make form
+		form := make(url.Values)
+		form.Add("page", fmt.Sprint(page))
+		form.Add("action", "filter")
+		form.Add("pageId", "7")
+		form.Add("key", parseKey)
+		// send request
+		err := client.
+			WithHeader("Cookie", fmt.Sprintf("PHPSESSID=%s", PHPSESSID)).
+			WithHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8").
+			Post("/assets/components/msearch2/action.php").WithForm(form).JSON(&resp)
+		if err != nil {
+			return 0, fmt.Errorf("%s: %w", op, err)
+		}
+		// update PHPSESSID and key
+		if resp.Message == "Could not load config" {
+			GetPHPSESSID(log)
+			GetKey(log)
+			continue
+		}
+		if !resp.Success {
+			err = errors.New("not success request")
+			return 0, fmt.Errorf("%s: %w", op, err)
+		}
+		body = strings.NewReader(resp.Data.Results)
+		break
+	}
+	doc, err := goquery.NewDocumentFromReader(body)
+	if err != nil {
+		return 0, fmt.Errorf("%s: %w", op, err)
+	}
+	var recipes []models.Recipe
+	doc.Find("div.recipe-card").Each(func(i int, s *goquery.Selection) {
+		var recipe models.Recipe
+		recipe.Title = s.Find("div.recipe-card__title").Text()
+		// recipe.Time = strings.ReplaceAll(strings.ReplaceAll(s.Find("div.recipe-card__time").Text(), "\n", ""), "  ", "")
+		recipe.Image, _ = s.Find("img").Attr("src")
+		recipe.Image = strings.Replace(recipe.Image, "image_366", "image_732", 1)
+		recipe.Image = fmt.Sprintf("%s%s", baseUrl, recipe.Image)
+		recipe.Link, _ = s.Find("a.recipe-card__link").Attr("href")
+		recipe.Link = fmt.Sprintf("/%s", recipe.Link)
+
+		recipes = append(recipes, recipe)
+	})
+
+	var wg sync.WaitGroup
+	wg.Add(len(recipes))
+	for i := 0; i < len(recipes); i++ {
+		go func(i int) {
+			defer wg.Done()
+			err = recipes[i].GetRecipe()
+			if err != nil {
+				return fmt.Errorf("%s: %w", op, err)
+			}
+		}(i)
+	}
+	wg.Wait()
+	return resp.Data.Pages, nil
+}
+
+// GetRecipe gets recipe info and saves recipe to storage.
+func GetRecipe(r *models.Recipe) error {
+	const op = "parser.GetRecipe"
+
+	if r.Link == "" {
+		return fmt.Errorf("%s: %w", op, errors.New("empty link"))
+	}
+	// send request
+	body, err := client.
+		WithHeader("Cookie", fmt.Sprintf("PHPSESSID=%s", PHPSESSID)).
+		Post(r.Link).String()
+	if err != nil {
+		return fmt.Errorf("%s: %w", op, err)
+	}
+	bodyr := strings.NewReader(body)
+	doc, err := goquery.NewDocumentFromReader(bodyr)
+	if err != nil {
+		return fmt.Errorf("%s: %w", op, err)
+	}
+	// описание
+	r.Description = strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(doc.Find("div.card-binfo__description").Text(), "  ", ""), "\n", ""), "\t", "")
+	// Время готовки, кол-во порций
+	doc.Find("div.recipe-summary-items>div.recipe-summary-item").Each(func(i int, s *goquery.Selection) {
+		label := s.Find("div.recipe-summary-item__label").Text()
+		value := strings.ReplaceAll(s.Find("div.recipe-summary-item__value").Text(), "\n", "")
+		switch label {
+		case "Время приготовления:":
+			r.CookingTime = value
+		case "Количество порций:":
+			count, _ := strconv.Atoi(value)
+			r.ServingsNum = uint(count)
+		case "Калорийность:":
+			r.Calories = value
+		}
+	})
+	doc.Find("div.ingredients").Each(func(i int, s *goquery.Selection) {
+		// ингридиенты для рецепта
+		var ingredients models.RecipeIngredients
+		ingredients.Title = s.Find("div.ingredients__title").Text()
+		s.Find("ul.ingredients__list>li").Each(func(i int, q *goquery.Selection) {
+			ingredient := strings.ReplaceAll(strings.ReplaceAll(q.Text(), "  ", ""), "\n", "")
+			ingredients.Ingredients = append(ingredients.Ingredients, ingredient)
+		})
+		if len(ingredients.Ingredients) != 0 {
+			r.Ingredients = append(r.Ingredients, ingredients)
+		}
+		// шаги рецепта
+		var recipe_steps []string
+		s.Find("div.recipe-rich>ol>li>span").Each(func(i int, q *goquery.Selection) {
+			recipe_step := strings.ReplaceAll(strings.ReplaceAll(q.Text(), "  ", ""), "\n", "")
+			recipe_steps = append(recipe_steps, recipe_step)
+		})
+		if len(recipe_steps) != 0 {
+			r.Recipe_steps = recipe_steps
+		}
+		// рекомендации
+		var advices []string
+		s.Find("div.recipe-footer__additional-text>p").Each(func(i int, q *goquery.Selection) {
+			if q.Find("br").Length() > 0 {
+				html, err := q.Html()
+				if err != nil {
+
+					return fmt.Errorf("%s: %w", op, err)
+				}
+				advice_list := q.SetHtml(strings.Replace(html, "<br/>", "\n", -1)).Text()
+				advices_arr := strings.Split(advice_list, "\n")
+				var advices_arr_res []string = make([]string, 0, len(advices_arr))
+				for _, a := range advices_arr {
+					if a != "" {
+						advices_arr_res = append(advices_arr_res, a)
+					}
+				}
+				advices = append(advices, advices_arr_res...)
+			} else {
+				var advice string = q.Text()
+				advices = append(advices, advice)
+			}
+		})
+		if len(advices) != 0 {
+			// fmt.Printf("LEN ADVICES = %d", len(advices))
+			r.Advices = advices
+		}
+	})
+	// категории
+	doc.Find("div.similar-items>a.similar-items__link").Each(func(i int, s *goquery.Selection) {
+		r.Categories = append(r.Categories, s.Text())
+	})
+	// // вывод результатов
+	// fmt.Println("-------------------")
+	// fmt.Printf("%+v\n", r)
+	// fmt.Println("-------------------")
+	// check recipe exists
+	ex, err := postgres.DB.RecipeExists(r.Title)
+	if err != nil || ex {
+		return fmt.Errorf("%s: %w", op, fmt.Errorf("recipe already exists"))
+	}
+	// save picture
+	err = r.SaveRecipePicture()
+	// add to database
+	var final_recipe models.Recipe = models.Recipe{
+		Title:        r.Title,
+		Description:  r.Description,
+		Image:        r.Image,
+		CookingTime:  r.CookingTime,
+		Link:         r.Link,
+		ServingsNum:  r.ServingsNum,
+		Calories:     r.Calories,
+		Ingredients:  r.Ingredients,
+		Recipe_steps: r.Recipe_steps,
+		Advices:      r.Advices,
+		Categories:   r.Categories,
+	}
+	// insert recipe
+	err = postgres.DB.AddRecipe(final_recipe)
+	return fmt.Errorf("%s: %w", op, err)
+}
+
+// func (r *Recipe) SaveRecipePicture() error {
+// 	resp, err := http.Get(r.Image)
+// 	if err != nil {
+
+// 		return err
+// 	}
+// 	defer resp.Body.Close()
+// 	content_len, _ := strconv.ParseInt(resp.Header["Content-Length"][0], 10, 64)
+// 	// change name to generated uuid
+// 	filename := renamefile(getFilenameFromUrl(r.Image), uuid.NewString())
+// 	// upload to minio
+// 	err = cminio.UploadFile(cminio.RecipeImg, filename, resp.Body, resp.Header["Content-Type"][0], content_len)
+
+// 	// change to filename
+// 	r.Image = filename
+// 	return err
+// }
+
+// // url to filename
+// func getFilenameFromUrl(url string) string {
+// 	url_els := strings.Split(url, "/")
+// 	return url_els[len(url_els)-1]
+// }
+
+// // change file name
+// func renamefile(old_filename, new_name string) string {
+// 	old_file_els := strings.Split(old_filename, ".")
+// 	return fmt.Sprintf("%s.%s", new_name, old_file_els[len(old_file_els)-1])
+// }
+
+// GetKey gets
+func GetKey(log slog.Logger) error {
+	const op = "parser.GetKey"
+
+	log.Debug("Updating KEY...")
+	defer log.Debug("KEY updated")
+	str, err := client.WithHeader("Cookie", fmt.Sprintf("PHPSESSID=%s", PHPSESSID)).Post("/recipes").String()
+	if err != nil {
+
+		return fmt.Errorf("%s: %w", op, err)
+	}
+	i := strings.Index(str, "\"key\":")
+	if i != 0 {
+		parseKey = str[i+7 : i+47]
+		log.Debug("New KEY =", parseKey)
+		return nil
+	}
+	err = errors.New("key not found")
+
+	return fmt.Errorf("%s: %w", op, err)
+}
+
+func GetPHPSESSID(log slog.Logger) error {
+	const op = "parser.GetPHPSESSID"
+
+	log.Debug("Updating PHPSESSID...")
+	defer log.Debug("PHPSESSID updated")
+	form := make(url.Values)
+	form.Add("page", "1")
+	form.Add("action", "filter")
+	form.Add("pageId", "7")
+	form.Add("key", "-")
+	resp, err := client.
+		WithHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8").
+		WithHeader("Cookie", "").
+		Post("/assets/components/msearch2/action.php").WithForm(form).Do()
+	if err != nil {
+		return fmt.Errorf("%s: %w", op, err)
+	}
+	for _, c := range resp.Response().Cookies() {
+		if c.Name == "PHPSESSID" {
+			PHPSESSID = c.Value
+			log.Debug("New PHPSESSID =", PHPSESSID)
+			return nil
+		}
+	}
+	err = errors.New("cookie not found")
+
+	return fmt.Errorf("%s: %w", op, err)
+}

internal/parser/structs.go

@@ -0,0 +1,14 @@
+package parser
+
+type GetPageResp struct {
+	Success bool   `json:"success"`
+	Message string `json:"message"`
+	Data    struct {
+		Results    string `json:"results"`
+		Pagination string `json:"pagination"`
+		Total      int    `json:"total"`
+		Page       int    `json:"page"`
+		Pages      int    `json:"pages"`
+		Log        string `json:"log"`
+	} `json:"data"`
+}

internal/storage/postgresql/postgresql.go

@@ -12,10 +12,10 @@ type Storage struct {
 	db *pgxpool.Pool
 }
 
-func New(user, password, addr, dbname string) (*Storage, error) {
+func New(ctx context.Context, user, password, addr, dbname string) (*Storage, error) {
 	const op = "storage.postgresql.New"
 
-	pool, err := pgxpool.New(context.Background(), fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", user, password, addr, dbname))
+	pool, err := pgxpool.New(ctx, fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", user, password, addr, dbname))
 	if err != nil {
 		return nil, fmt.Errorf("%s: %w", op, err)
 	}