From 7759ada8e09cea922d1f3d7bcdfcf5667d77deea Mon Sep 17 00:00:00 2001 From: Li Zhe Date: Sun, 7 Apr 2024 16:46:16 +0800 Subject: [PATCH 01/11] only response in --- search4all.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/search4all.py b/search4all.py index 9e1df03..03d3fc1 100644 --- a/search4all.py +++ b/search4all.py @@ -62,7 +62,7 @@ You are a large language AI assistant built by AI. You are given a user question Your answer must be correct, accurate and written by an expert using an unbiased and professional tone. Please limit to 1024 tokens. Do not give any information that is not related to the question, and do not repeat. Say "information is missing on" followed by the related topic, if the given context do not provide sufficient information. -Please cite the contexts with the reference numbers, in the format [citation:x]. If a sentence comes from multiple contexts, please list all applicable citations, like [citation:3][citation:5]. Other than code and specific names and citations, your answer must be written in the same language as the question. +Please cite the contexts with the reference numbers, in the format [citation:x]. If a sentence comes from multiple contexts, please list all applicable citations, like [citation:3][citation:5]. Other than code and specific names and citations, your answer must be written in Chinese（中文）. Here are the set of contexts: @@ -907,4 +907,4 @@ app.static("/", os.path.join(BASE_DIR, "ui/index.html"), name="ui") if __name__ == "__main__": port = int(os.getenv("PORT") or 8800) workers = int(os.getenv("WORKERS") or 1) - app.run(host="0.0.0.0", port=port, workers=workers, debug=False) \ No newline at end of file + app.run(host="0.0.0.0", port=port, workers=workers, debug=False) From bd46bea0cf56a1450227fa5fa06f4a9300fdbb37 Mon Sep 17 00:00:00 2001 From: Li Zhe Date: Sun, 7 Apr 2024 18:58:38 +0800 Subject: [PATCH 02/11] support searxng --- requirements.txt | 5 ++- search4all.py | 93 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5c684e0..66e30b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,7 @@ anthropic loguru sanic sqlitedict -python-dotenv \ No newline at end of file +python-dotenv +tld==0.13 +tldextract==5.1.2 +trafilatura==1.8.1 \ No newline at end of file diff --git a/search4all.py b/search4all.py index 03d3fc1..7a95372 100644 --- a/search4all.py +++ b/search4all.py @@ -11,6 +11,12 @@ import asyncio from anthropic import AsyncAnthropic from loguru import logger from dotenv import load_dotenv +import urllib.parse +import trafilatura +from trafilatura import bare_extraction +import tldextract +from concurrent.futures import ThreadPoolExecutor +from urllib.parse import urlparse load_dotenv() import sanic @@ -378,6 +384,88 @@ def search_with_searchapi(query: str, subscription_key: str): return [] +def extract_url_content(url): + downloaded = trafilatura.fetch_url(url) + content = trafilatura.extract(downloaded) + + return {"url":url, "content":content} + + + +def search_with_searXNG(query:str,url:str, debug=False): + + content_list = [] + + try: + + safe_string = urllib.parse.quote_plus(":all !general " + query) + response = requests.get(url+'?q=' + safe_string + '&format=json') + response.raise_for_status() + search_results = response.json() + + if debug: + print("JSON Response:") + logger.info(search_results) + pedding_urls = [] + + conv_links = [] + + if search_results.get('results'): + for item in search_results.get('results')[0:9]: + name = item.get('title') + snippet = item.get('content') + url = item.get('url') + pedding_urls.append(url) + + if url: + url_parsed = urlparse(url) + domain = url_parsed.netloc + icon_url = url_parsed.scheme + '://' + url_parsed.netloc + '/favicon.ico' + site_name = tldextract.extract(url).domain + + conv_links.append({ + 'site_name':site_name, + 'icon_url':icon_url, + 'title':name, + 'url':url, + 'snippet':snippet + }) + + results = [] + futures = [] + + executor = ThreadPoolExecutor(max_workers=10) + for url in pedding_urls: + futures.append(executor.submit(extract_url_content,url)) + try: + for future in futures: + res = future.result(timeout=5) + results.append(res) + except concurrent.futures.TimeoutError: + print("任务执行超时") + executor.shutdown(wait=False,cancel_futures=True) + + for content in results: + if content and content.get('content'): + + item_dict = { + "url":content.get('url'), + "name":content.get('url'), + "snippet":content.get('content'), + "content": content.get('content'), + "length":len(content.get('content')) + } + content_list.append(item_dict) + if debug: + print("URL: {}".format(url)) + print("=================") + + return content_list + except Exception as ex: + raise ex + + + def new_async_client(_app): if "claude-3" in _app.ctx.model.lower(): return AsyncAnthropic( @@ -436,6 +524,11 @@ async def server_init(_app): query, _app.ctx.search1api_key, ) + elif _app.ctx.backend == "SEARXNG": + _app.ctx.search_function = lambda query: search_with_searXNG( + query, + url = os.getenv("SEARXNG_BASE_URL"), + ) else: raise RuntimeError("Backend must be BING, GOOGLE, SERPER or SEARCHAPI or SEARCH1API.") _app.ctx.model = os.getenv("LLM_MODEL") From 2ce67d3e6f5c17c0b43569d754c20e507ba46952 Mon Sep 17 00:00:00 2001 From: Li Zhe Date: Sun, 7 Apr 2024 19:15:31 +0800 Subject: [PATCH 03/11] support searxng --- search4all.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/search4all.py b/search4all.py index 7a95372..b1d2732 100644 --- a/search4all.py +++ b/search4all.py @@ -392,7 +392,7 @@ def extract_url_content(url): -def search_with_searXNG(query:str,url:str, debug=False): +def search_with_searXNG(query:str,url:str): content_list = [] @@ -403,9 +403,9 @@ def search_with_searXNG(query:str,url:str, debug=False): response.raise_for_status() search_results = response.json() - if debug: - print("JSON Response:") - logger.info(search_results) + + logger.info("JSON Response:") + logger.info(search_results) pedding_urls = [] conv_links = [] @@ -442,7 +442,7 @@ def search_with_searXNG(query:str,url:str, debug=False): res = future.result(timeout=5) results.append(res) except concurrent.futures.TimeoutError: - print("任务执行超时") + logger.error("任务执行超时") executor.shutdown(wait=False,cancel_futures=True) for content in results: @@ -456,12 +456,12 @@ def search_with_searXNG(query:str,url:str, debug=False): "length":len(content.get('content')) } content_list.append(item_dict) - if debug: - print("URL: {}".format(url)) - print("=================") - + print("URL: {}".format(url)) + print("=================") + return content_list except Exception as ex: + logger.error(ex) raise ex @@ -525,9 +525,10 @@ async def server_init(_app): _app.ctx.search1api_key, ) elif _app.ctx.backend == "SEARXNG": + logger.info(os.getenv("SEARXNG_BASE_URL")) _app.ctx.search_function = lambda query: search_with_searXNG( query, - url = os.getenv("SEARXNG_BASE_URL"), + os.getenv("SEARXNG_BASE_URL"), ) else: raise RuntimeError("Backend must be BING, GOOGLE, SERPER or SEARCHAPI or SEARCH1API.") From 2f854703ea1b270fc25510dc257199560f96aeb6 Mon Sep 17 00:00:00 2001 From: Li Zhe Date: Sun, 7 Apr 2024 19:33:27 +0800 Subject: [PATCH 04/11] support searxng --- search4all.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/search4all.py b/search4all.py index b1d2732..c7873b1 100644 --- a/search4all.py +++ b/search4all.py @@ -387,7 +387,8 @@ def search_with_searchapi(query: str, subscription_key: str): def extract_url_content(url): downloaded = trafilatura.fetch_url(url) content = trafilatura.extract(downloaded) - + logger.info(url) + logger.info(content) return {"url":url, "content":content} @@ -422,7 +423,7 @@ def search_with_searXNG(query:str,url:str): domain = url_parsed.netloc icon_url = url_parsed.scheme + '://' + url_parsed.netloc + '/favicon.ico' site_name = tldextract.extract(url).domain - + conv_links.append({ 'site_name':site_name, 'icon_url':icon_url, @@ -430,7 +431,7 @@ def search_with_searXNG(query:str,url:str): 'url':url, 'snippet':snippet }) - + logger.info(conv_links) results = [] futures = [] @@ -444,7 +445,7 @@ def search_with_searXNG(query:str,url:str): except concurrent.futures.TimeoutError: logger.error("任务执行超时") executor.shutdown(wait=False,cancel_futures=True) - + logger.info(results) for content in results: if content and content.get('content'): @@ -456,8 +457,8 @@ def search_with_searXNG(query:str,url:str): "length":len(content.get('content')) } content_list.append(item_dict) - print("URL: {}".format(url)) - print("=================") + logger.info("URL: {}".format(url)) + logger.info("=================") return content_list except Exception as ex: From 607030140e33e7d12afa03e2834234bb907a90a0 Mon Sep 17 00:00:00 2001 From: Li Zhe Date: Sun, 7 Apr 2024 20:07:23 +0800 Subject: [PATCH 05/11] support serxng --- search4all.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/search4all.py b/search4all.py index c7873b1..337b03e 100644 --- a/search4all.py +++ b/search4all.py @@ -385,10 +385,11 @@ def search_with_searchapi(query: str, subscription_key: str): def extract_url_content(url): + logger.info(url) downloaded = trafilatura.fetch_url(url) content = trafilatura.extract(downloaded) - logger.info(url) - logger.info(content) + + logger.info(url +"______"+ content) return {"url":url, "content":content} @@ -399,8 +400,8 @@ def search_with_searXNG(query:str,url:str): try: - safe_string = urllib.parse.quote_plus(":all !general " + query) - response = requests.get(url+'?q=' + safe_string + '&format=json') + safe_string = urllib.parse.quote_plus(":auto !general " + query) + response = requests.get(url+'?q=' + safe_string + '&format=json&engines=bing,google') response.raise_for_status() search_results = response.json() @@ -428,6 +429,7 @@ def search_with_searXNG(query:str,url:str): 'site_name':site_name, 'icon_url':icon_url, 'title':name, + 'name':name, 'url':url, 'snippet':snippet }) @@ -459,7 +461,8 @@ def search_with_searXNG(query:str,url:str): content_list.append(item_dict) logger.info("URL: {}".format(url)) logger.info("=================") - + if len(results)== 0 : + content_list = conv_links return content_list except Exception as ex: logger.error(ex) From 5f9191027c795f746ace2ac635c355b602cfe26b Mon Sep 17 00:00:00 2001 From: Li Zhe Date: Mon, 8 Apr 2024 09:36:00 +0800 Subject: [PATCH 06/11] constrain bing and google --- search4all.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/search4all.py b/search4all.py index 337b03e..4e07d62 100644 --- a/search4all.py +++ b/search4all.py @@ -399,13 +399,12 @@ def search_with_searXNG(query:str,url:str): content_list = [] try: - + query="除夕放假么？" safe_string = urllib.parse.quote_plus(":auto !general " + query) - response = requests.get(url+'?q=' + safe_string + '&format=json&engines=bing,google') + response = requests.get(url+'?q=' + safe_string + '&format=json&engines=bing%2Cgoogle') response.raise_for_status() search_results = response.json() - - + logger.info("JSON Response:") logger.info(search_results) pedding_urls = [] From 060b6b67b86e9f59d1169dd9cb086cab43807d87 Mon Sep 17 00:00:00 2001 From: Li Zhe Date: Mon, 8 Apr 2024 09:44:49 +0800 Subject: [PATCH 07/11] no fetch the page --- search4all.py | 47 +++++++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/search4all.py b/search4all.py index 4e07d62..0fc1531 100644 --- a/search4all.py +++ b/search4all.py @@ -399,7 +399,6 @@ def search_with_searXNG(query:str,url:str): content_list = [] try: - query="除夕放假么？" safe_string = urllib.parse.quote_plus(":auto !general " + query) response = requests.get(url+'?q=' + safe_string + '&format=json&engines=bing%2Cgoogle') response.raise_for_status() @@ -436,30 +435,30 @@ def search_with_searXNG(query:str,url:str): results = [] futures = [] - executor = ThreadPoolExecutor(max_workers=10) - for url in pedding_urls: - futures.append(executor.submit(extract_url_content,url)) - try: - for future in futures: - res = future.result(timeout=5) - results.append(res) - except concurrent.futures.TimeoutError: - logger.error("任务执行超时") - executor.shutdown(wait=False,cancel_futures=True) - logger.info(results) - for content in results: - if content and content.get('content'): + # executor = ThreadPoolExecutor(max_workers=10) + # for url in pedding_urls: + # futures.append(executor.submit(extract_url_content,url)) + # try: + # for future in futures: + # res = future.result(timeout=5) + # results.append(res) + # except concurrent.futures.TimeoutError: + # logger.error("任务执行超时") + # executor.shutdown(wait=False,cancel_futures=True) + # logger.info(results) + # for content in results: + # if content and content.get('content'): - item_dict = { - "url":content.get('url'), - "name":content.get('url'), - "snippet":content.get('content'), - "content": content.get('content'), - "length":len(content.get('content')) - } - content_list.append(item_dict) - logger.info("URL: {}".format(url)) - logger.info("=================") + # item_dict = { + # "url":content.get('url'), + # "name":content.get('url'), + # "snippet":content.get('content'), + # "content": content.get('content'), + # "length":len(content.get('content')) + # } + # content_list.append(item_dict) + # logger.info("URL: {}".format(url)) + # logger.info("=================") if len(results)== 0 : content_list = conv_links return content_list From 44fc96c9d47d334e12789c40c4777af776d20b95 Mon Sep 17 00:00:00 2001 From: Li Zhe Date: Mon, 8 Apr 2024 09:49:46 +0800 Subject: [PATCH 08/11] change category filter --- search4all.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/search4all.py b/search4all.py index 0fc1531..9104e5e 100644 --- a/search4all.py +++ b/search4all.py @@ -399,8 +399,8 @@ def search_with_searXNG(query:str,url:str): content_list = [] try: - safe_string = urllib.parse.quote_plus(":auto !general " + query) - response = requests.get(url+'?q=' + safe_string + '&format=json&engines=bing%2Cgoogle') + safe_string = urllib.parse.quote_plus(":auto " + query) + response = requests.get(url+'?q=' + safe_string + '&category=general&format=json&engines=bing%2Cgoogle') response.raise_for_status() search_results = response.json() From 223e1d7549a0ca845cb3749e9e1161e1074b3a86 Mon Sep 17 00:00:00 2001 From: Li Zhe Date: Tue, 9 Apr 2024 14:29:59 +0800 Subject: [PATCH 09/11] provide the document --- README.md | 9 ++++++--- search4all.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6c62511..4c8204b 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,9 @@ To use the Bing Web Search API, please visit [this link](https://www.microsoft.c ### Google Search You have three options for Google Search: you can use the [SearchApi Google Search API](https://www.searchapi.io/) from SearchApi, [Serper Google Search API](https://www.serper.dev) from Serper, or opt for the [Programmable Search Engine](https://developers.google.com/custom-search) provided by Google. +### SearXNG Search +you can host your personal [SearXNG server](https://github.com/searxng/searxng), then you do not need pay for the search api. You just need provide the server address in `SEARXNG_BASE_URL`, with [aurora](https://github.com/aurora-develop/aurora),you can have a free private ai search engine. Be sure you enable the json format for the SearXNG server. + ## Deployment ### Zeabur Just click on it @@ -92,7 +95,7 @@ This project provides some additional configuration items set with environment v | `LLM_MODEL` | Yes | The model you want to use,support all chat models of openai, groq and claude. | `gpt-3.5-turbo-0125,mixtral-8x7b-32768,claude-3-haiku-20240307...` | `RELATED_QUESTIONS` | No | Show the related questions. | `1` | `NODE_ENV` | No | The environment required for deployment is necessary only during manual deployment. | `production` -| `BACKEND` | Yes | The search service you want. | `SEARCH1API,BING,GOOGLE,SERPER,SEARCHAPI` +| `BACKEND` | Yes | The search service you want. | `SEARCH1API,BING,GOOGLE,SERPER,SEARCHAPI,SEARXNG` | `CHAT_HISTORY` | No | Continue to ask about the results | `1` | `SEARCH1API_KEY` | Yes | If you choose SEARCH1API. | `xxx` | `BING_SEARCH_V7_SUBSCRIPTION_KEY` | No | If you choose BING. | `xxx` @@ -100,8 +103,8 @@ This project provides some additional configuration items set with environment v | `GOOGLE_SEARCH_API_KEY` | No | If you choose GOOGLE. | `xxx` | `SEARCHAPI_API_KEY` | No | If you choose SEARCHAPI. | `xxx` | `SERPER_SEARCH_API_KEY` | No | If you choose SERPER. | `xxx` -| `NEXT_PUBLIC_GOOGLE_ANALYTICS` | No | You can use Google Analytics to know how many users you have on your website. | MEASUREMENT ID,you can find on your google analytics account,like `G-XXXXXX` - +| `NEXT_PUBLIC_GOOGLE_ANALYTICS` | No | You can use Google Analytics to know how many users you have on your website. | MEASUREMENT ID,you can find on your google analytics account,like `G-XXXXXX` +| `SEARXNG_BASE_URL` | No | the hosted serxng server address. it is required when the BACKEND is `SEARXNG` | `https://serxng.xxx.com/` diff --git a/search4all.py b/search4all.py index 9104e5e..2a8b5d1 100644 --- a/search4all.py +++ b/search4all.py @@ -68,7 +68,7 @@ You are a large language AI assistant built by AI. You are given a user question Your answer must be correct, accurate and written by an expert using an unbiased and professional tone. Please limit to 1024 tokens. Do not give any information that is not related to the question, and do not repeat. Say "information is missing on" followed by the related topic, if the given context do not provide sufficient information. -Please cite the contexts with the reference numbers, in the format [citation:x]. If a sentence comes from multiple contexts, please list all applicable citations, like [citation:3][citation:5]. Other than code and specific names and citations, your answer must be written in Chinese（中文）. +Please cite the contexts with the reference numbers, in the format [citation:x]. If a sentence comes from multiple contexts, please list all applicable citations, like [citation:3][citation:5]. Other than code and specific names and citations, your answer must be written in the same language as the question. Here are the set of contexts: From 3d4c3005bf60e5931b24b5c4fad41b82d127df36 Mon Sep 17 00:00:00 2001 From: Li Zhe Date: Tue, 9 Apr 2024 14:42:23 +0800 Subject: [PATCH 10/11] limit the max_tokens for related_question --- search4all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search4all.py b/search4all.py index 2a8b5d1..c431505 100644 --- a/search4all.py +++ b/search4all.py @@ -599,7 +599,7 @@ async def get_related_questions(_app, query, contexts): response = await client.beta.tools.messages.create( model=_app.ctx.model, system=_more_questions_prompt, - max_tokens=4096, + max_tokens=1000, tools=tools, messages=[ {"role": "user", "content": query}, From 12247024ad4b9fca9b196872cb8df7f03d9da997 Mon Sep 17 00:00:00 2001 From: Li Zhe Date: Tue, 9 Apr 2024 14:57:05 +0800 Subject: [PATCH 11/11] limit the max_tokens for related_questions --- search4all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search4all.py b/search4all.py index c431505..b3e2459 100644 --- a/search4all.py +++ b/search4all.py @@ -657,7 +657,7 @@ async def get_related_questions(_app, query, contexts): request_body = { "model": _app.ctx.model, "messages": messages, - "max_tokens": 4096, + "max_tokens": 1000, "tools": tools, "tool_choice": { "type": "function",