原代码:
while True:
conn, cur_pgsql = self.connectPGSQL()
sql = "SELECT id,file_path FROM listed_companies.company_new_notice WHERE file_path_ is null and id > {0} limit 20".format(index)
cur_pgsql.execute(sql)
items = cur_pgsql.fetchall()
cur_pgsql.close()
conn.close()
with ThreadPoolExecutor(max_workers=20) as pool:
pool.map(self.downloadPDF, items)
index += 20
运行后没反应,Ctrl+c后就有了如下报错信息
Traceback (most recent call last):
File "company_new_notice.py", line 224, in <module>
spider.downloadUrl()
File "company_new_notice.py", line 218, in downloadUrl
pool.map(self.downloadPDF, items)
File "/data01/python3/lib/python3.6/concurrent/futures/_base.py", line 581, in __exit__
self.shutdown(wait=True)
File "/data01/python3/lib/python3.6/concurrent/futures/thread.py", line 144, in shutdown
t.join()
File "/data01/python3/lib/python3.6/threading.py", line 1056, in join
self._wait_for_tstate_lock()
File "/data01/python3/lib/python3.6/threading.py", line 1072, in _wait_for_tstate_lock
elif lock.acquire(block, timeout):
KeyboardInterrupt
^CError in atexit._run_exitfuncs:
Traceback (most recent call last):
File "/data01/python3/lib/python3.6/concurrent/futures/thread.py", line 39, in _python_exit
t.join()
File "/data01/python3/lib/python3.6/threading.py", line 1056, in join
self._wait_for_tstate_lock()
File "/data01/python3/lib/python3.6/threading.py", line 1072, in _wait_for_tstate_lock
elif lock.acquire(block, timeout):
KeyboardInterrupt