searpl

commit c62b48ae1f52c3f1529ed2a41d3f35fe705f3b21
parent 69de9f49dcd5d6d604620bafe406de7146d51450
Author: xfnw <xfnw@thunix.net>
Date:   Mon, 14 Dec 2020 17:22:28 -0500

unique urls

Diffstat:
M
create.php
 | 
2
+-
M
urls.sh
 | 
4
++--

2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/create.php b/create.php
@@ -1,2 +1,2 @@
-CREATE TABLE indexed (id INTEGER PRIMARY KEY, title VARCHAR(255), url VARCHAR(512), content TEXT)
+CREATE TABLE indexed (id INTEGER PRIMARY KEY, title VARCHAR(255), url VARCHAR(512) UNIQUE, content TEXT)
 
diff --git a/urls.sh b/urls.sh
@@ -1,3 +1,3 @@
-wget --spider --force-html -r -l2 -H $@ 2>&1 \
+wget --spider --force-html -r -l1 -H $@ 2>&1 \
   | grep '^--' | awk '{ print $3 }' \
-  | grep -v '\.\(css\|js\|png\|gif\|jpg\)$'
+  | grep -v '\.\(css\|js\|png\|gif\|jpg\|txt\|ico\|ttf\)$'