use goquery instead of colly
This commit is contained in:
Aleksandr Trushkin
12
go.mod
12
go.mod
@ -4,11 +4,11 @@ go 1.21.4
|
||||
|
||||
require (
|
||||
github.com/BurntSushi/toml v1.3.2
|
||||
github.com/PuerkitoBio/goquery v1.8.1
|
||||
github.com/brianvoe/gofakeit/v6 v6.28.0
|
||||
github.com/dgraph-io/badger/v4 v4.2.0
|
||||
github.com/dgraph-io/ristretto v0.1.1
|
||||
github.com/go-resty/resty/v2 v2.10.0
|
||||
github.com/gocolly/colly v1.2.0
|
||||
github.com/google/flatbuffers v23.5.26+incompatible
|
||||
github.com/rodaine/table v1.1.1
|
||||
github.com/rs/zerolog v1.31.0
|
||||
@ -17,31 +17,21 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.8.1 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.1 // indirect
|
||||
github.com/antchfx/htmlquery v1.3.0 // indirect
|
||||
github.com/antchfx/xmlquery v1.3.18 // indirect
|
||||
github.com/antchfx/xpath v1.2.4 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.0 // indirect
|
||||
github.com/gobwas/glob v0.2.3 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/glog v1.0.0 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/golang/snappy v0.0.3 // indirect
|
||||
github.com/kennygrant/sanitize v1.2.4 // indirect
|
||||
github.com/klauspost/compress v1.12.3 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
|
||||
github.com/temoto/robotstxt v1.1.2 // indirect
|
||||
github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect
|
||||
go.opencensus.io v0.22.5 // indirect
|
||||
golang.org/x/net v0.17.0 // indirect
|
||||
golang.org/x/sys v0.13.0 // indirect
|
||||
golang.org/x/text v0.13.0 // indirect
|
||||
google.golang.org/appengine v1.4.0 // indirect
|
||||
google.golang.org/protobuf v1.28.1 // indirect
|
||||
)
|
||||
|
||||
24
go.sum
24
go.sum
@ -6,13 +6,6 @@ github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAc
|
||||
github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
|
||||
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
|
||||
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||
github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E=
|
||||
github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8=
|
||||
github.com/antchfx/xmlquery v1.3.18 h1:FSQ3wMuphnPPGJOFhvc+cRQ2CT/rUj4cyQXkJcjOwz0=
|
||||
github.com/antchfx/xmlquery v1.3.18/go.mod h1:Afkq4JIeXut75taLSuI31ISJ/zeq+3jG7TunF7noreA=
|
||||
github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
|
||||
github.com/antchfx/xpath v1.2.4 h1:dW1HB/JxKvGtJ9WyVGJ0sIoEcqftV3SqIstujI+B9XY=
|
||||
github.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
|
||||
github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4=
|
||||
github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
@ -33,10 +26,6 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/go-resty/resty/v2 v2.10.0 h1:Qla4W/+TMmv0fOeeRqzEpXPLfTUnR5HZ1+lGs+CkiCo=
|
||||
github.com/go-resty/resty/v2 v2.10.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
|
||||
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
|
||||
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
|
||||
github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
|
||||
github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
@ -60,8 +49,6 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
|
||||
github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU=
|
||||
@ -84,21 +71,16 @@ github.com/rodaine/table v1.1.1/go.mod h1:iqTRptjn+EVcrVBYtNMlJ2wrJZa3MpULUmcXFp
|
||||
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
|
||||
github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
|
||||
github.com/samber/do v1.6.0 h1:Jy/N++BXINDB6lAx5wBlbpHlUdl0FKpLWgGEV9YWqaU=
|
||||
github.com/samber/do v1.6.0/go.mod h1:DWqBvumy8dyb2vEnYZE7D7zaVEB64J45B0NjTlY/M4k=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
|
||||
github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
|
||||
github.com/urfave/cli/v3 v3.0.0-alpha8 h1:H+qxFPoCkGzdF8KUMs2fEOZl5io/1QySgUiGfar8occ=
|
||||
github.com/urfave/cli/v3 v3.0.0-alpha8/go.mod h1:0kK/RUFHyh+yIKSfWxwheGndfnrvYSmYFVeKCh03ZUc=
|
||||
github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e h1:+SOyEddqYF09QP7vr7CgJ1eti3pY9Fn3LHO1M1r/0sI=
|
||||
@ -132,7 +114,6 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
@ -159,7 +140,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@ -168,7 +148,6 @@ golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
|
||||
@ -176,10 +155,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
|
||||
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
@ -197,7 +174,6 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
|
||||
@ -18,8 +18,8 @@ import (
|
||||
"git.loyso.art/frx/eway/internal/crypto"
|
||||
"git.loyso.art/frx/eway/internal/entity"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/go-resty/resty/v2"
|
||||
"github.com/gocolly/colly"
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
@ -191,12 +191,12 @@ func (c *client) GetGoodsRemnants(
|
||||
productsStr = append(productsStr, strconv.Itoa(sku))
|
||||
}
|
||||
|
||||
resp, err := c.http.R().
|
||||
req := c.http.R().
|
||||
SetFormData(map[string]string{
|
||||
"products": strings.Join(productsStr, ","),
|
||||
}).
|
||||
SetDoNotParseResponse(true).
|
||||
Post("/goods_remnants")
|
||||
SetDoNotParseResponse(true)
|
||||
resp, err := c.do(ctx, "GetGoodsRemnants", req, resty.MethodPost, "/goods_remnants")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting goods new: %w", err)
|
||||
}
|
||||
@ -230,22 +230,36 @@ func (c *client) GetGoodsNew(
|
||||
params GetGoodsNewParams,
|
||||
) (items []entity.GoodsItemRaw, total int, err error) {
|
||||
var response getGoodsNewResponse
|
||||
resp, err := c.http.R().
|
||||
SetFormData(map[string]string{
|
||||
"draw": strconv.Itoa(params.Draw),
|
||||
"start": strconv.Itoa(params.Start),
|
||||
"length": strconv.Itoa(params.Length),
|
||||
"order[0][column]": "14",
|
||||
"order[0][dir]": "desc",
|
||||
"search[value]": "",
|
||||
"search[regex]": "false",
|
||||
"search_in_stocks": "on",
|
||||
"remnants_atleast": "5",
|
||||
}).
|
||||
formData := map[string]string{
|
||||
"draw": strconv.Itoa(params.Draw),
|
||||
"start": strconv.Itoa(params.Start),
|
||||
"length": strconv.Itoa(params.Length),
|
||||
"order[0][column]": "14",
|
||||
"order[0][dir]": "desc",
|
||||
"search[value]": "",
|
||||
"search[regex]": "false",
|
||||
}
|
||||
if params.SearchInStocks {
|
||||
stocksNum := strconv.Itoa(params.RemmantsAtleast)
|
||||
formData["search_in_stocks"] = "on"
|
||||
formData["remnants_atleast"] = stocksNum
|
||||
}
|
||||
|
||||
c.log.Debug().
|
||||
Int("remnants", params.RemmantsAtleast).
|
||||
Bool("search_in_stocks", params.SearchInStocks).
|
||||
Int("draw", params.Draw).
|
||||
Int("start", params.Start).
|
||||
Int("length", params.Length).
|
||||
Msg("sending request")
|
||||
|
||||
req := c.http.R().
|
||||
SetFormData(formData).
|
||||
SetQueryParam("category_id", "0").
|
||||
SetQueryParam("own", c.ownerID). // user id?
|
||||
SetDoNotParseResponse(true).
|
||||
Post("/goods_new")
|
||||
SetDoNotParseResponse(true)
|
||||
|
||||
resp, err := c.do(ctx, "GetGoodsNew", req, resty.MethodPost, "/goods_new")
|
||||
if err != nil {
|
||||
return nil, -1, fmt.Errorf("getting goods new: %w", err)
|
||||
}
|
||||
@ -255,7 +269,6 @@ func (c *client) GetGoodsNew(
|
||||
c.log.Error().Err(err).Msg("unable to close body")
|
||||
}
|
||||
}()
|
||||
|
||||
if resp.IsError() {
|
||||
return nil, -1, errors.New("request was not successful")
|
||||
}
|
||||
@ -269,12 +282,14 @@ func (c *client) GetGoodsNew(
|
||||
}
|
||||
|
||||
func (c *client) login(ctx context.Context, user, pass string) error {
|
||||
resp, err := c.http.R().
|
||||
req := c.http.R().
|
||||
SetDoNotParseResponse(true).
|
||||
SetFormData(map[string]string{
|
||||
"username": user,
|
||||
"password": pass,
|
||||
}).Post("https://eway.elevel.ru/")
|
||||
})
|
||||
|
||||
resp, err := c.do(ctx, "login", req, resty.MethodPost, "https://eway.elevel.ru/")
|
||||
if err != nil {
|
||||
return fmt.Errorf("sending request: %w", err)
|
||||
}
|
||||
@ -288,14 +303,27 @@ func (c *client) login(ctx context.Context, user, pass string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type ProductInfo struct {
|
||||
ImageLinks []string
|
||||
Parameters map[string]string
|
||||
}
|
||||
func (c *client) do(ctx context.Context, name string, req *resty.Request, method string, url string) (resp *resty.Response, err error) {
|
||||
resp, err = req.
|
||||
EnableTrace().
|
||||
Execute(method, url)
|
||||
|
||||
type parameterSelector struct {
|
||||
Name string `selector:"div"`
|
||||
Value string `selector:"div.text-right"`
|
||||
traceInfo := resp.Request.TraceInfo()
|
||||
c.log.Debug().
|
||||
Str("name", name).
|
||||
Str("path", url).
|
||||
Str("method", method).
|
||||
Float64("elapsed", traceInfo.TotalTime.Seconds()).
|
||||
Float64("response_time", traceInfo.ResponseTime.Seconds()).
|
||||
Int("attempt", traceInfo.RequestAttempt).
|
||||
Bool("success", resp.IsSuccess()).
|
||||
Msg("request processed")
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("executing request: %w", err)
|
||||
}
|
||||
|
||||
return resp, err
|
||||
}
|
||||
|
||||
func (c *client) GetProductInfo(ctx context.Context, cart int64) (pi entity.GoodsItemInfo, err error) {
|
||||
@ -317,69 +345,73 @@ func (c *client) GetProductInfo(ctx context.Context, cart int64) (pi entity.Good
|
||||
}
|
||||
}
|
||||
|
||||
func (c *client) getProductInfo(ctx context.Context, cart int64) (pi entity.GoodsItemInfo, err error) {
|
||||
collector := colly.NewCollector(
|
||||
colly.AllowedDomains("eway.elevel.ru"),
|
||||
colly.AllowURLRevisit(),
|
||||
)
|
||||
func (c *client) getProductInfo(ctx context.Context, cartID int64) (pi entity.GoodsItemInfo, err error) {
|
||||
reqpath := "https://eway.elevel.ru/product/" + strconv.Itoa(int(cartID)) + "/"
|
||||
|
||||
req := c.http.R().SetDoNotParseResponse(true).AddRetryCondition(func(r *resty.Response, err error) bool {
|
||||
if r.Request.Attempt > 3 {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(err.Error(), "pipe")
|
||||
})
|
||||
|
||||
c.log.Debug().Msg("using go query")
|
||||
|
||||
pi.Parameters = map[string]string{}
|
||||
|
||||
start := time.Now()
|
||||
resp, err := c.do(ctx, "getProductInfo", req, resty.MethodGet, reqpath)
|
||||
if err != nil {
|
||||
return pi, fmt.Errorf("getting product info: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
elapsed := time.Since(start).Seconds()
|
||||
c.log.Info().
|
||||
Float64("elapsed", elapsed).
|
||||
Int64("cart", cart).
|
||||
Msg("request processed")
|
||||
errClose := resp.RawBody().Close()
|
||||
if errClose == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
err = errClose
|
||||
return
|
||||
}
|
||||
|
||||
c.log.Warn().Err(errClose).Msg("unable to close body")
|
||||
}()
|
||||
if resp.IsError() {
|
||||
return pi, errors.New("request was not successful")
|
||||
}
|
||||
|
||||
collector.OnHTML("body > div.page-container > div.page-content > div.content-wrapper > div.content > div.row > div.col-md-4 > div > div > div:nth-child(6)", func(e *colly.HTMLElement) {
|
||||
e.ForEach("div.display-flex", func(i int, h *colly.HTMLElement) {
|
||||
var s parameterSelector
|
||||
err = h.Unmarshal(&s)
|
||||
if err != nil {
|
||||
c.log.Warn().Err(err).Msg("unable to unmarshal")
|
||||
return
|
||||
}
|
||||
doc, err := goquery.NewDocumentFromReader(resp.RawBody())
|
||||
if err != nil {
|
||||
return pi, fmt.Errorf("makind new document: %w", err)
|
||||
}
|
||||
|
||||
if s.Name == "" || s.Value == "" {
|
||||
c.log.Warn().Msg("got empty key or value, skipping")
|
||||
return
|
||||
}
|
||||
cleanText := func(t string) string {
|
||||
return strings.TrimSuffix(strings.TrimSpace(t), ":")
|
||||
}
|
||||
|
||||
pi.Parameters[s.Name] = s.Value
|
||||
const parametersSelector = "body > div.page-container > div.page-content > div.content-wrapper > div.content > div.row > div.col-md-4 > div > div > div:nth-child(6)"
|
||||
const parametersInnerNode = "div.display-flex"
|
||||
doc.
|
||||
Find(parametersSelector).
|
||||
Find(parametersInnerNode).
|
||||
Each(func(i int, s *goquery.Selection) {
|
||||
name := cleanText(s.Find("div").Eq(0).Text())
|
||||
value := cleanText(s.Find("div.text-right").Text())
|
||||
pi.Parameters[name] = value
|
||||
})
|
||||
})
|
||||
collector.OnHTML("div.gallery_panel", func(h *colly.HTMLElement) {
|
||||
h.ForEach("div.gallery_thumbnail > img", func(i int, h *colly.HTMLElement) {
|
||||
imageURL := h.Attr("src")
|
||||
|
||||
if imageURL == "" {
|
||||
const galleryPanelSelector = "div.gallery_panel"
|
||||
const galleryImageSelector = "div.gallery_thumbnail > img"
|
||||
doc.
|
||||
Find(galleryPanelSelector).
|
||||
Find(galleryImageSelector).
|
||||
Each(func(i int, s *goquery.Selection) {
|
||||
imageURL, ok := s.Attr("src")
|
||||
if !ok || len(imageURL) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
pi.PhotoURLs = append(pi.PhotoURLs, imageURL)
|
||||
})
|
||||
})
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
err = collector.Visit("https://eway.elevel.ru/product/" + strconv.Itoa(int(cart)) + "/")
|
||||
if err != nil {
|
||||
c.log.Warn().Err(err).Msg("unable to visit site, retrying...")
|
||||
select {
|
||||
case <-time.After(time.Second * 2):
|
||||
continue
|
||||
case <-ctx.Done():
|
||||
return pi, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return pi, fmt.Errorf("visiting site: %w", err)
|
||||
}
|
||||
|
||||
return pi, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user