Skip to content

Commit

Permalink
small refactor
Browse files Browse the repository at this point in the history
  • Loading branch information
Mzack9999 committed Oct 31, 2023
1 parent 6d0c3ca commit 19db3cf
Show file tree
Hide file tree
Showing 3 changed files with 6 additions and 4 deletions.
3 changes: 1 addition & 2 deletions pkg/engine/common/base.go
Original file line number Diff line number Diff line change
Expand Up @@ -239,8 +239,7 @@ func (s *Shared) Do(crawlSession *CrawlSession, doRequest DoRequestFunc) error {
if resp.Resp == nil || resp.Reader == nil {
return
}
isRedirectResponse := resp.StatusCode >= 300 && resp.StatusCode < 400
if s.Options.Options.DisableRedirects && isRedirectResponse {
if s.Options.Options.DisableRedirects && resp.IsRedirect() {
return
}

Expand Down
3 changes: 1 addition & 2 deletions pkg/engine/hybrid/crawl.go
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,7 @@ func (c *Crawler) navigateRequest(s *common.CrawlSession, request *navigation.Re
c.Enqueue(s.Queue, navigationRequests...)

// do not continue following the request if it's a redirect and redirects are disabled
isRedirectResponse := statusCode >= 300 && statusCode < 400
if c.Options.Options.DisableRedirects && isRedirectResponse {
if c.Options.Options.DisableRedirects && resp.IsRedirect() {
return nil
}
return FetchContinueRequest(page, e)
Expand Down
4 changes: 4 additions & 0 deletions pkg/navigation/response.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,3 +58,7 @@ func (n Response) AbsoluteURL(path string) string {
final := absURL.String()
return final
}

func (n Response) IsRedirect() bool {
return n.StatusCode >= 300 && n.StatusCode <= 399
}

0 comments on commit 19db3cf

Please sign in to comment.