From 632a84057237dedd3b6b407d17ed02889d519dae Mon Sep 17 00:00:00 2001
From: Mahdi Dibaiee <mdibaiee@aol.com>
Date: Sat, 22 Apr 2017 13:21:43 +0430
Subject: [PATCH] fix: memory improvement

---
 index.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/index.py b/index.py
index 788b4de..dc021b6 100644
--- a/index.py
+++ b/index.py
@@ -79,7 +79,7 @@ for host in sites:
             
             if href and href != '/':
                 p = to_absolute(href, host)
-                if p and p.netloc == main.netloc:
+                if p and p.netloc == main.netloc and p.geturl() not in visited:
                     queue.insert(0, p)