Patch: update crawling to not follow redirects when -disable-redirects is set

pkg/engine/common/base.go

@@ -239,6 +239,9 @@ func (s *Shared) Do(crawlSession *CrawlSession, doRequest DoRequestFunc) error {
 			if resp.Resp == nil || resp.Reader == nil {
 				return
 			}
+			if s.Options.Options.DisableRedirects && resp.IsRedirect() {
+				return
+			}
 
 			navigationRequests := parser.ParseResponse(resp)
 			s.Enqueue(crawlSession.Queue, navigationRequests...)

pkg/engine/hybrid/crawl.go

@@ -128,6 +128,11 @@ func (c *Crawler) navigateRequest(s *common.CrawlSession, request *navigation.Re
 		// process the raw response
 		navigationRequests := parser.ParseResponse(resp)
 		c.Enqueue(s.Queue, navigationRequests...)
+
+		// do not continue following the request if it's a redirect and redirects are disabled
+		if c.Options.Options.DisableRedirects && resp.IsRedirect() {
+			return nil
+		}
 		return FetchContinueRequest(page, e)
 	})() //nolint
 	defer func() {

pkg/engine/parser/parser.go

@@ -37,7 +37,6 @@ var responseParsers = []responseParser{
 	// Header based parsers
 	{headerParser, headerContentLocationParser},
 	{headerParser, headerLinkParser},
-	{headerParser, headerLocationParser},
 	{headerParser, headerRefreshParser},
 
 	// Body based parsers
@@ -84,6 +83,9 @@ func InitWithOptions(options *types.Options) {
 		responseParsers = append(responseParsers, responseParser{contentParser, scriptJSFileRegexParser})
 		responseParsers = append(responseParsers, responseParser{contentParser, bodyScrapeEndpointsParser})
 	}
+	if !options.DisableRedirects {
+		responseParsers = append(responseParsers, responseParser{headerParser, headerLocationParser})
+	}
 }
 
 // parseResponse runs the response parsers on the navigation response

pkg/navigation/response.go

@@ -58,3 +58,7 @@ func (n Response) AbsoluteURL(path string) string {
 	final := absURL.String()
 	return final
 }
+
+func (n Response) IsRedirect() bool {
+	return n.StatusCode >= 300 && n.StatusCode <= 399
+}