多线程的小爬虫，测试后发现有三条内容重复了 40 多次

<pre class="brush:python;toolbar:false">import requests
from bs4 import BeautifulSoup
import threading
url_num = 0
url_list = [&#39;http://ubuntuforums.org/forumdisplay.php?f=333&#39;,]
for x in range(1, 50):
url_num += 1
raw_url = &#39;http://ubuntuforums.org/forumdisplay.php?f=333&amp;page=%d&#39; % url_num
url_list.append(raw_url)
class MyThread(threading.Thread):
def __init__(self, func, args, name=&quot;&quot;):
threading.Thread.__init__(self)
self.func = func
self.args = args
self.name = name
def run(self):
apply(self.func, self.args)
def running(url):
# lock.acquire()
html = requests.get(url)
if html.status_code == 200:
html_text = html.text
soup = BeautifulSoup(html_text)
with open(&#39;/home/zhg/Pictures/cao.txt&#39;, &#39;a+&#39;) as f:
for link in soup.find_all(&#39;a&#39;, &#39;title&#39;):
s = &#39;http://ubuntuforums.org/&#39; + str(link.get(&#39;href&#39;)) + &#39; &#39; + str(link.get_text().encode(&#39;utf-8&#39;))
f.writelines(s)
f.writelines(&#39;\n&#39;)
# lock.release()
if __name__ == &#39;__main__&#39;:
thread_list = [ MyThread(running, (url, ), running.__name__) for url in url_list ]
for t in thread_list:
t.setDaemon(True)
t.start()
for i in thread_list:
i.join()
print &quot;process ended&quot;
with open(&#39;/home/zhg/Pictures/cao.txt&#39;, &#39;r&#39;) as f:
f_list = f.readlines()
set_list = set(f_list)
for x in set_list:
if f_list.count(x) &gt; 1:
print &quot;the &lt;%s&gt; has found &lt;%d&gt;&quot; % (x, f_list.count(x))</pre>
<p>
<br/>
</p>




结果：
process ended
the <http://ubuntuforums.org/showthread.php?t=2229766&s=bb6cd917fa0c28a6e9cb02be35fa4379 Forums Staff recommendations on WUBI
> has found <49>
the <http://ubuntuforums.org/showthread.php?t=1946145&s=bb6cd917fa0c28a6e9cb02be35fa4379 I upgraded, and now I have this error...
> has found <49>
the <http://ubuntuforums.org/showthread.php?t=1743535&s=bb6cd917fa0c28a6e9cb02be35fa4379 Graphics Resolution- Upgrade /Blank Screen after reboot
> has found <49>