From ccc668105abca9e409f73ad7164ff9641367bc1d Mon Sep 17 00:00:00 2001 From: Aleksandr Trushkin Date: Tue, 6 Feb 2024 19:14:55 +0300 Subject: [PATCH] use goquery instead of colly --- go.mod | 12 +- go.sum | 24 ---- internal/interconnect/eway/client.go | 186 ++++++++++++++++----------- 3 files changed, 110 insertions(+), 112 deletions(-) diff --git a/go.mod b/go.mod index b8f3249..9d9d434 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,11 @@ go 1.21.4 require ( github.com/BurntSushi/toml v1.3.2 + github.com/PuerkitoBio/goquery v1.8.1 github.com/brianvoe/gofakeit/v6 v6.28.0 github.com/dgraph-io/badger/v4 v4.2.0 github.com/dgraph-io/ristretto v0.1.1 github.com/go-resty/resty/v2 v2.10.0 - github.com/gocolly/colly v1.2.0 github.com/google/flatbuffers v23.5.26+incompatible github.com/rodaine/table v1.1.1 github.com/rs/zerolog v1.31.0 @@ -17,31 +17,21 @@ require ( ) require ( - github.com/PuerkitoBio/goquery v1.8.1 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect - github.com/antchfx/htmlquery v1.3.0 // indirect - github.com/antchfx/xmlquery v1.3.18 // indirect - github.com/antchfx/xpath v1.2.4 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/gobwas/glob v0.2.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.3 // indirect - github.com/kennygrant/sanitize v1.2.4 // indirect github.com/klauspost/compress v1.12.3 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect - github.com/temoto/robotstxt v1.1.2 // indirect github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect go.opencensus.io v0.22.5 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect - google.golang.org/appengine v1.4.0 // indirect google.golang.org/protobuf v1.28.1 // indirect ) diff --git a/go.sum b/go.sum index 304ce75..c087921 100644 --- a/go.sum +++ b/go.sum @@ -6,13 +6,6 @@ github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAc github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= -github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E= -github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8= -github.com/antchfx/xmlquery v1.3.18 h1:FSQ3wMuphnPPGJOFhvc+cRQ2CT/rUj4cyQXkJcjOwz0= -github.com/antchfx/xmlquery v1.3.18/go.mod h1:Afkq4JIeXut75taLSuI31ISJ/zeq+3jG7TunF7noreA= -github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= -github.com/antchfx/xpath v1.2.4 h1:dW1HB/JxKvGtJ9WyVGJ0sIoEcqftV3SqIstujI+B9XY= -github.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4= github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -33,10 +26,6 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/go-resty/resty/v2 v2.10.0 h1:Qla4W/+TMmv0fOeeRqzEpXPLfTUnR5HZ1+lGs+CkiCo= github.com/go-resty/resty/v2 v2.10.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= -github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= -github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -60,8 +49,6 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= -github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= @@ -84,21 +71,16 @@ github.com/rodaine/table v1.1.1/go.mod h1:iqTRptjn+EVcrVBYtNMlJ2wrJZa3MpULUmcXFp github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= -github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= -github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/samber/do v1.6.0 h1:Jy/N++BXINDB6lAx5wBlbpHlUdl0FKpLWgGEV9YWqaU= github.com/samber/do v1.6.0/go.mod h1:DWqBvumy8dyb2vEnYZE7D7zaVEB64J45B0NjTlY/M4k= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= -github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/urfave/cli/v3 v3.0.0-alpha8 h1:H+qxFPoCkGzdF8KUMs2fEOZl5io/1QySgUiGfar8occ= github.com/urfave/cli/v3 v3.0.0-alpha8/go.mod h1:0kK/RUFHyh+yIKSfWxwheGndfnrvYSmYFVeKCh03ZUc= github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e h1:+SOyEddqYF09QP7vr7CgJ1eti3pY9Fn3LHO1M1r/0sI= @@ -132,7 +114,6 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= @@ -159,7 +140,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -168,7 +148,6 @@ golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= @@ -176,10 +155,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -197,7 +174,6 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= diff --git a/internal/interconnect/eway/client.go b/internal/interconnect/eway/client.go index 9e26749..f43ea93 100644 --- a/internal/interconnect/eway/client.go +++ b/internal/interconnect/eway/client.go @@ -18,8 +18,8 @@ import ( "git.loyso.art/frx/eway/internal/crypto" "git.loyso.art/frx/eway/internal/entity" + "github.com/PuerkitoBio/goquery" "github.com/go-resty/resty/v2" - "github.com/gocolly/colly" "github.com/rs/zerolog" ) @@ -191,12 +191,12 @@ func (c *client) GetGoodsRemnants( productsStr = append(productsStr, strconv.Itoa(sku)) } - resp, err := c.http.R(). + req := c.http.R(). SetFormData(map[string]string{ "products": strings.Join(productsStr, ","), }). - SetDoNotParseResponse(true). - Post("/goods_remnants") + SetDoNotParseResponse(true) + resp, err := c.do(ctx, "GetGoodsRemnants", req, resty.MethodPost, "/goods_remnants") if err != nil { return nil, fmt.Errorf("getting goods new: %w", err) } @@ -230,22 +230,36 @@ func (c *client) GetGoodsNew( params GetGoodsNewParams, ) (items []entity.GoodsItemRaw, total int, err error) { var response getGoodsNewResponse - resp, err := c.http.R(). - SetFormData(map[string]string{ - "draw": strconv.Itoa(params.Draw), - "start": strconv.Itoa(params.Start), - "length": strconv.Itoa(params.Length), - "order[0][column]": "14", - "order[0][dir]": "desc", - "search[value]": "", - "search[regex]": "false", - "search_in_stocks": "on", - "remnants_atleast": "5", - }). + formData := map[string]string{ + "draw": strconv.Itoa(params.Draw), + "start": strconv.Itoa(params.Start), + "length": strconv.Itoa(params.Length), + "order[0][column]": "14", + "order[0][dir]": "desc", + "search[value]": "", + "search[regex]": "false", + } + if params.SearchInStocks { + stocksNum := strconv.Itoa(params.RemmantsAtleast) + formData["search_in_stocks"] = "on" + formData["remnants_atleast"] = stocksNum + } + + c.log.Debug(). + Int("remnants", params.RemmantsAtleast). + Bool("search_in_stocks", params.SearchInStocks). + Int("draw", params.Draw). + Int("start", params.Start). + Int("length", params.Length). + Msg("sending request") + + req := c.http.R(). + SetFormData(formData). SetQueryParam("category_id", "0"). SetQueryParam("own", c.ownerID). // user id? - SetDoNotParseResponse(true). - Post("/goods_new") + SetDoNotParseResponse(true) + + resp, err := c.do(ctx, "GetGoodsNew", req, resty.MethodPost, "/goods_new") if err != nil { return nil, -1, fmt.Errorf("getting goods new: %w", err) } @@ -255,7 +269,6 @@ func (c *client) GetGoodsNew( c.log.Error().Err(err).Msg("unable to close body") } }() - if resp.IsError() { return nil, -1, errors.New("request was not successful") } @@ -269,12 +282,14 @@ func (c *client) GetGoodsNew( } func (c *client) login(ctx context.Context, user, pass string) error { - resp, err := c.http.R(). + req := c.http.R(). SetDoNotParseResponse(true). SetFormData(map[string]string{ "username": user, "password": pass, - }).Post("https://eway.elevel.ru/") + }) + + resp, err := c.do(ctx, "login", req, resty.MethodPost, "https://eway.elevel.ru/") if err != nil { return fmt.Errorf("sending request: %w", err) } @@ -288,14 +303,27 @@ func (c *client) login(ctx context.Context, user, pass string) error { return nil } -type ProductInfo struct { - ImageLinks []string - Parameters map[string]string -} +func (c *client) do(ctx context.Context, name string, req *resty.Request, method string, url string) (resp *resty.Response, err error) { + resp, err = req. + EnableTrace(). + Execute(method, url) -type parameterSelector struct { - Name string `selector:"div"` - Value string `selector:"div.text-right"` + traceInfo := resp.Request.TraceInfo() + c.log.Debug(). + Str("name", name). + Str("path", url). + Str("method", method). + Float64("elapsed", traceInfo.TotalTime.Seconds()). + Float64("response_time", traceInfo.ResponseTime.Seconds()). + Int("attempt", traceInfo.RequestAttempt). + Bool("success", resp.IsSuccess()). + Msg("request processed") + + if err != nil { + return nil, fmt.Errorf("executing request: %w", err) + } + + return resp, err } func (c *client) GetProductInfo(ctx context.Context, cart int64) (pi entity.GoodsItemInfo, err error) { @@ -317,69 +345,73 @@ func (c *client) GetProductInfo(ctx context.Context, cart int64) (pi entity.Good } } -func (c *client) getProductInfo(ctx context.Context, cart int64) (pi entity.GoodsItemInfo, err error) { - collector := colly.NewCollector( - colly.AllowedDomains("eway.elevel.ru"), - colly.AllowURLRevisit(), - ) +func (c *client) getProductInfo(ctx context.Context, cartID int64) (pi entity.GoodsItemInfo, err error) { + reqpath := "https://eway.elevel.ru/product/" + strconv.Itoa(int(cartID)) + "/" + + req := c.http.R().SetDoNotParseResponse(true).AddRetryCondition(func(r *resty.Response, err error) bool { + if r.Request.Attempt > 3 { + return false + } + return strings.Contains(err.Error(), "pipe") + }) + + c.log.Debug().Msg("using go query") pi.Parameters = map[string]string{} - - start := time.Now() + resp, err := c.do(ctx, "getProductInfo", req, resty.MethodGet, reqpath) + if err != nil { + return pi, fmt.Errorf("getting product info: %w", err) + } defer func() { - elapsed := time.Since(start).Seconds() - c.log.Info(). - Float64("elapsed", elapsed). - Int64("cart", cart). - Msg("request processed") + errClose := resp.RawBody().Close() + if errClose == nil { + return + } + + if err == nil { + err = errClose + return + } + + c.log.Warn().Err(errClose).Msg("unable to close body") }() + if resp.IsError() { + return pi, errors.New("request was not successful") + } - collector.OnHTML("body > div.page-container > div.page-content > div.content-wrapper > div.content > div.row > div.col-md-4 > div > div > div:nth-child(6)", func(e *colly.HTMLElement) { - e.ForEach("div.display-flex", func(i int, h *colly.HTMLElement) { - var s parameterSelector - err = h.Unmarshal(&s) - if err != nil { - c.log.Warn().Err(err).Msg("unable to unmarshal") - return - } + doc, err := goquery.NewDocumentFromReader(resp.RawBody()) + if err != nil { + return pi, fmt.Errorf("makind new document: %w", err) + } - if s.Name == "" || s.Value == "" { - c.log.Warn().Msg("got empty key or value, skipping") - return - } + cleanText := func(t string) string { + return strings.TrimSuffix(strings.TrimSpace(t), ":") + } - pi.Parameters[s.Name] = s.Value + const parametersSelector = "body > div.page-container > div.page-content > div.content-wrapper > div.content > div.row > div.col-md-4 > div > div > div:nth-child(6)" + const parametersInnerNode = "div.display-flex" + doc. + Find(parametersSelector). + Find(parametersInnerNode). + Each(func(i int, s *goquery.Selection) { + name := cleanText(s.Find("div").Eq(0).Text()) + value := cleanText(s.Find("div.text-right").Text()) + pi.Parameters[name] = value }) - }) - collector.OnHTML("div.gallery_panel", func(h *colly.HTMLElement) { - h.ForEach("div.gallery_thumbnail > img", func(i int, h *colly.HTMLElement) { - imageURL := h.Attr("src") - if imageURL == "" { + const galleryPanelSelector = "div.gallery_panel" + const galleryImageSelector = "div.gallery_thumbnail > img" + doc. + Find(galleryPanelSelector). + Find(galleryImageSelector). + Each(func(i int, s *goquery.Selection) { + imageURL, ok := s.Attr("src") + if !ok || len(imageURL) == 0 { return } pi.PhotoURLs = append(pi.PhotoURLs, imageURL) }) - }) - - for i := 0; i < 3; i++ { - err = collector.Visit("https://eway.elevel.ru/product/" + strconv.Itoa(int(cart)) + "/") - if err != nil { - c.log.Warn().Err(err).Msg("unable to visit site, retrying...") - select { - case <-time.After(time.Second * 2): - continue - case <-ctx.Done(): - return pi, ctx.Err() - } - } - - break - } - if err != nil { - return pi, fmt.Errorf("visiting site: %w", err) - } return pi, nil }