auto increase depth when -kf is used (#655)

* auto increase depth when `-kf` is used * move to `validateOptions` func * updating readme --------- Co-authored-by: Mzack9999 <[email protected]>
projectdiscovery · Nov 30, 2023 · 80dd3d0 · 80dd3d0
1 parent 54891e9
commit 80dd3d0
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 4 deletions.
diff --git a/README.md b/README.md
@@ -115,14 +115,15 @@ Usage:
 Flags:
 INPUT:
    -u, -list string[]  target url / list to crawl
+   -resume string      resume scan using resume.cfg
 
 CONFIGURATION:
    -r, -resolvers string[]       list of custom resolver (file or comma separated)
    -d, -depth int                maximum depth to crawl (default 3)
    -jc, -js-crawl                enable endpoint parsing / crawling in javascript file
-   -jsl, -jsluice                 enable jsluice parsing in javascript file (memory intensive)
+   -jsl, -jsluice                enable jsluice parsing in javascript file (memory intensive)
    -ct, -crawl-duration value    maximum duration to crawl the target for (s, m, h, d) (default s)
-   -kf, -known-files string      enable crawling of known files (all,robotstxt,sitemapxml)
+   -kf, -known-files string      enable crawling of known files (all,robotstxt,sitemapxml), a minimum depth of 3 is required to ensure all known files are properly crawled.
    -mrs, -max-response-size int  maximum response size to read (default 9223372036854775807)
    -timeout int                  time to wait for request in seconds (default 10)
    -aff, -automatic-form-fill    enable automatic form filling (experimental)
@@ -136,6 +137,7 @@ CONFIGURATION:
    -s, -strategy string          Visit strategy (depth-first, breadth-first) (default "depth-first")
    -iqp, -ignore-query-params    Ignore crawling same path with different query-param values
    -tlsi, -tls-impersonate       enable experimental client hello (ja3) tls randomization
+   -dr, -disable-redirects       disable following redirects (default false)
 
 DEBUG:
    -health-check, -hc        run diagnostic check up
@@ -156,7 +158,7 @@ HEADLESS:
 SCOPE:
    -cs, -crawl-scope string[]       in scope url regex to be followed by crawler
    -cos, -crawl-out-scope string[]  out of scope url regex to be excluded by crawler
-   -fs, -field-scope string  pre-defined scope field (dn,rdn,fqdn) or custom regex (e.g., '(company-staging.io|company.com)') (default "rdn")
+   -fs, -field-scope string         pre-defined scope field (dn,rdn,fqdn) or custom regex (e.g., '(company-staging.io|company.com)') (default "rdn")
    -ns, -no-scope                   disables host based default scope
    -do, -display-out-scope          display external endpoint from scoped crawling
 

diff --git a/cmd/katana/main.go b/cmd/katana/main.go
@@ -91,7 +91,7 @@ pipelines offering both headless and non-headless crawling.`)
 		flagSet.BoolVarP(&options.ScrapeJSResponses, "js-crawl", "jc", false, "enable endpoint parsing / crawling in javascript file"),
 		flagSet.BoolVarP(&options.ScrapeJSLuiceResponses, "jsluice", "jsl", false, "enable jsluice parsing in javascript file (memory intensive)"),
 		flagSet.DurationVarP(&options.CrawlDuration, "crawl-duration", "ct", 0, "maximum duration to crawl the target for (s, m, h, d) (default s)"),
-		flagSet.StringVarP(&options.KnownFiles, "known-files", "kf", "", "enable crawling of known files (all,robotstxt,sitemapxml)"),
+		flagSet.StringVarP(&options.KnownFiles, "known-files", "kf", "", "enable crawling of known files (all,robotstxt,sitemapxml), a minimum depth of 3 is required to ensure all known files are properly crawled."),
 		flagSet.IntVarP(&options.BodyReadSize, "max-response-size", "mrs", math.MaxInt, "maximum response size to read"),
 		flagSet.IntVar(&options.Timeout, "timeout", 10, "time to wait for request in seconds"),
 		flagSet.BoolVarP(&options.AutomaticFormFill, "automatic-form-fill", "aff", false, "enable automatic form filling (experimental)"),
@@ -182,6 +182,7 @@ pipelines offering both headless and non-headless crawling.`)
 			return nil, errorutil.NewWithErr(err).Msgf("could not read config file")
 		}
 	}
+
 	cleanupOldResumeFiles()
 	return flagSet, nil
 }

diff --git a/internal/runner/options.go b/internal/runner/options.go
@@ -52,6 +52,10 @@ func validateOptions(options *types.Options) error {
 		}
 		options.FilterRegex = append(options.FilterRegex, cr)
 	}
+	if options.KnownFiles != "" && options.MaxDepth < 3 {
+		gologger.Info().Msgf("Depth automatically set to 3 to accommodate the `--known-files` option (originally set to %d).", options.MaxDepth)
+		options.MaxDepth = 3
+	}
 	gologger.DefaultLogger.SetFormatter(formatter.NewCLI(options.NoColors))
 	return nil
 }