commit c6b476c4a35ec4e9b79ee2c9839f9b85cc93ae27 parent 3e051c0febb56a1d2c0e47fee5430f3856a9c655 Author: xfnw <xfnw@thunix.net> Date: Sat, 16 Jan 2021 13:37:01 -0500 crawl slower so sites wont ban it lol Diffstat:
| M | urls.sh | | | 2 | +- |
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/urls.sh b/urls.sh @@ -1,4 +1,4 @@ -wget --spider --force-html --tries 1 --timeout 1 -r -l1 -H -U 'searplbot/1.0' $@ 2>&1 | tee -a wg +wget -w 2 --random-wait --spider --force-html --tries 1 --timeout 1 -r -l1 -H -U 'searplbot/1.0' $@ 2>&1 | tee -a wg grep '^--' wg | awk '{ print $3 }' \ | grep -v '\.\(css\|js\|png\|gif\|jpg\|txt\|ico\|ttf\|svg\)$' \