home/SAI2
2
0
Fork 0
mirror of https://github.com/wahyd4/SAI2.git synced 2026-08-08 21:01:21 +10:00
Code Issues Packages Projects Releases Wiki Activity

no fetch the page

Browse Source
This commit is contained in:
Li Zhe
2024-04-08 09:44:49 +08:00
parent 5f9191027c
commit 060b6b67b8
1 changed files with 23 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
search4all.py
+23 -24
View File
@@ -399,7 +399,6 @@ def search_with_searXNG(query:str,url:str):
content_list = []
try:
query="除夕放假么？"
safe_string = urllib.parse.quote_plus(":auto !general " + query)
response = requests.get(url+'?q=' + safe_string + '&format=json&engines=bing%2Cgoogle')
response.raise_for_status()
@@ -436,30 +435,30 @@ def search_with_searXNG(query:str,url:str):
results = []
futures = []
executor = ThreadPoolExecutor(max_workers=10)
for url in pedding_urls:
futures.append(executor.submit(extract_url_content,url))
try:
for future in futures:
res = future.result(timeout=5)
results.append(res)
except concurrent.futures.TimeoutError:
logger.error("任务执行超时")
executor.shutdown(wait=False,cancel_futures=True)
logger.info(results)
for content in results:
if content and content.get('content'):
# executor = ThreadPoolExecutor(max_workers=10)
# for url in pedding_urls:
# futures.append(executor.submit(extract_url_content,url))
# try:
# for future in futures:
# res = future.result(timeout=5)
# results.append(res)
# except concurrent.futures.TimeoutError:
# logger.error("任务执行超时")
# executor.shutdown(wait=False,cancel_futures=True)
# logger.info(results)
# for content in results:
# if content and content.get('content'):
item_dict = {
"url":content.get('url'),
"name":content.get('url'),
"snippet":content.get('content'),
"content": content.get('content'),
"length":len(content.get('content'))
}
content_list.append(item_dict)
logger.info("URL: {}".format(url))
logger.info("=================")
# item_dict = {
# "url":content.get('url'),
# "name":content.get('url'),
# "snippet":content.get('content'),
# "content": content.get('content'),
# "length":len(content.get('content'))
# }
# content_list.append(item_dict)
# logger.info("URL: {}".format(url))
# logger.info("=================")
if len(results)== 0 :
content_list = conv_links
return content_list