added flag to skip storing responses

README.md

@@ -99,6 +99,7 @@ Options:
   -j, --javascript <string> JavaScript to run on each page
   -o, --output <string>     Output directory name (default 'out')
   -p, --proxy <string>      Use proxy on given URL
+  -s, --skip-save-response  Do not save responses in general
   -w, --overwrite           Overwrite output files when they already exist
       --no-third-party      Do not save responses to requests on third-party domains
       --third-party         Only save responses to requests on third-party domains

main.go

@@ -35,6 +35,7 @@ func init() {
 			"  -j, --javascript <string> JavaScript to run on each page",
 			"  -o, --output <string>     Output directory name (default 'out')",
 			"  -p, --proxy <string>      Use proxy on given URL",
+			"  -s, --skip-save-response  Do not save responses in general",
 			"  -w, --overwrite           Overwrite output files when they already exist",
 			"      --no-third-party      Do not save responses to requests on third-party domains",
 			"      --third-party         Only save responses to requests on third-party domains",
@@ -46,15 +47,16 @@ func init() {
 }
 
 type options struct {
-	includes       listArg
-	excludes       listArg
-	thirdPartyOnly bool
-	noThirdParty   bool
-	overwrite      bool
-	output         string
-	concurrency    int
-	js             string
-	proxy          string
+	includes         listArg
+	excludes         listArg
+	thirdPartyOnly   bool
+	noThirdParty     bool
+	overwrite        bool
+	skipSaveResponse bool
+	output           string
+	concurrency      int
+	js               string
+	proxy            string
 }
 
 func main() {
@@ -67,6 +69,9 @@ func main() {
 	flag.Var(&opts.excludes, "exclude", "")
 	flag.Var(&opts.excludes, "e", "")
 
+	flag.BoolVar(&opts.skipSaveResponse, "skip-save-response", false, "")
+	flag.BoolVar(&opts.skipSaveResponse, "s", false, "")
+
 	flag.BoolVar(&opts.thirdPartyOnly, "third-party", false, "")
 	flag.BoolVar(&opts.noThirdParty, "no-third-party", false, "")
 
@@ -336,26 +341,27 @@ func makeListener(ctx context.Context, requestURL string, opts options) func(int
 				err := chromedp.Run(
 					ctx,
 					chromedp.ActionFunc(func(ctx context.Context) error {
-						data, err := fetch.GetResponseBody(ev.RequestID).Do(ctx)
-						if err != nil {
-							// this function always has to return a nil error
-							// otherwise the ContinueRequest does not run
-							return nil
-						}
-
-						path, err := saveResponse(ev.Request.URL, data, opts.output, opts.overwrite)
-						if err != nil {
-							fmt.Fprintf(os.Stderr, "failed to save response data for %s: %s\n", ev.Request.URL, err)
-							return nil
+						if !opts.skipSaveResponse {
+							data, err := fetch.GetResponseBody(ev.RequestID).Do(ctx)
+							if err != nil {
+								// this function always has to return a nil error
+								// otherwise the ContinueRequest does not run
+								return nil
+							}
+
+							path, err := saveResponse(ev.Request.URL, data, opts.output, opts.overwrite)
+							if err != nil {
+								fmt.Fprintf(os.Stderr, "failed to save response data for %s: %s\n", ev.Request.URL, err)
+								return nil
+							}
+
+							// save the headers etc in a separate file
+							err = saveMeta(path+".meta", requestURL, ev)
+							if err != nil {
+								fmt.Fprintf(os.Stderr, "failed to save response meta data for %s: %s\n", ev.Request.URL, err)
+								return nil
+							}
 						}
-
-						// save the headers etc in a separate file
-						err = saveMeta(path+".meta", requestURL, ev)
-						if err != nil {
-							fmt.Fprintf(os.Stderr, "failed to save response meta data for %s: %s\n", ev.Request.URL, err)
-							return nil
-						}
-
 						// Log the request
 						fmt.Printf("%s %s %d %s\n", ev.Request.Method, ev.Request.URL, ev.ResponseStatusCode, contentType)