diff --git a/README.md b/README.md index 6c62511..4c8204b 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,9 @@ To use the Bing Web Search API, please visit [this link](https://www.microsoft.c ### Google Search You have three options for Google Search: you can use the [SearchApi Google Search API](https://www.searchapi.io/) from SearchApi, [Serper Google Search API](https://www.serper.dev) from Serper, or opt for the [Programmable Search Engine](https://developers.google.com/custom-search) provided by Google. +### SearXNG Search +you can host your personal [SearXNG server](https://github.com/searxng/searxng), then you do not need pay for the search api. You just need provide the server address in `SEARXNG_BASE_URL`, with [aurora](https://github.com/aurora-develop/aurora),you can have a free private ai search engine. Be sure you enable the json format for the SearXNG server. + ## Deployment ### Zeabur Just click on it @@ -92,7 +95,7 @@ This project provides some additional configuration items set with environment v | `LLM_MODEL` | Yes | The model you want to use,support all chat models of openai, groq and claude. | `gpt-3.5-turbo-0125,mixtral-8x7b-32768,claude-3-haiku-20240307...` | `RELATED_QUESTIONS` | No | Show the related questions. | `1` | `NODE_ENV` | No | The environment required for deployment is necessary only during manual deployment. | `production` -| `BACKEND` | Yes | The search service you want. | `SEARCH1API,BING,GOOGLE,SERPER,SEARCHAPI` +| `BACKEND` | Yes | The search service you want. | `SEARCH1API,BING,GOOGLE,SERPER,SEARCHAPI,SEARXNG` | `CHAT_HISTORY` | No | Continue to ask about the results | `1` | `SEARCH1API_KEY` | Yes | If you choose SEARCH1API. | `xxx` | `BING_SEARCH_V7_SUBSCRIPTION_KEY` | No | If you choose BING. | `xxx` @@ -100,8 +103,8 @@ This project provides some additional configuration items set with environment v | `GOOGLE_SEARCH_API_KEY` | No | If you choose GOOGLE. | `xxx` | `SEARCHAPI_API_KEY` | No | If you choose SEARCHAPI. | `xxx` | `SERPER_SEARCH_API_KEY` | No | If you choose SERPER. | `xxx` -| `NEXT_PUBLIC_GOOGLE_ANALYTICS` | No | You can use Google Analytics to know how many users you have on your website. | MEASUREMENT ID,you can find on your google analytics account,like `G-XXXXXX` - +| `NEXT_PUBLIC_GOOGLE_ANALYTICS` | No | You can use Google Analytics to know how many users you have on your website. | MEASUREMENT ID,you can find on your google analytics account,like `G-XXXXXX` +| `SEARXNG_BASE_URL` | No | the hosted serxng server address. it is required when the BACKEND is `SEARXNG` | `https://serxng.xxx.com/` diff --git a/requirements.txt b/requirements.txt index 5c684e0..66e30b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,7 @@ anthropic loguru sanic sqlitedict -python-dotenv \ No newline at end of file +python-dotenv +tld==0.13 +tldextract==5.1.2 +trafilatura==1.8.1 \ No newline at end of file diff --git a/search4all.py b/search4all.py index 9e1df03..b3e2459 100644 --- a/search4all.py +++ b/search4all.py @@ -11,6 +11,12 @@ import asyncio from anthropic import AsyncAnthropic from loguru import logger from dotenv import load_dotenv +import urllib.parse +import trafilatura +from trafilatura import bare_extraction +import tldextract +from concurrent.futures import ThreadPoolExecutor +from urllib.parse import urlparse load_dotenv() import sanic @@ -378,6 +384,90 @@ def search_with_searchapi(query: str, subscription_key: str): return [] +def extract_url_content(url): + logger.info(url) + downloaded = trafilatura.fetch_url(url) + content = trafilatura.extract(downloaded) + + logger.info(url +"______"+ content) + return {"url":url, "content":content} + + + +def search_with_searXNG(query:str,url:str): + + content_list = [] + + try: + safe_string = urllib.parse.quote_plus(":auto " + query) + response = requests.get(url+'?q=' + safe_string + '&category=general&format=json&engines=bing%2Cgoogle') + response.raise_for_status() + search_results = response.json() + + logger.info("JSON Response:") + logger.info(search_results) + pedding_urls = [] + + conv_links = [] + + if search_results.get('results'): + for item in search_results.get('results')[0:9]: + name = item.get('title') + snippet = item.get('content') + url = item.get('url') + pedding_urls.append(url) + + if url: + url_parsed = urlparse(url) + domain = url_parsed.netloc + icon_url = url_parsed.scheme + '://' + url_parsed.netloc + '/favicon.ico' + site_name = tldextract.extract(url).domain + + conv_links.append({ + 'site_name':site_name, + 'icon_url':icon_url, + 'title':name, + 'name':name, + 'url':url, + 'snippet':snippet + }) + logger.info(conv_links) + results = [] + futures = [] + + # executor = ThreadPoolExecutor(max_workers=10) + # for url in pedding_urls: + # futures.append(executor.submit(extract_url_content,url)) + # try: + # for future in futures: + # res = future.result(timeout=5) + # results.append(res) + # except concurrent.futures.TimeoutError: + # logger.error("任务执行超时") + # executor.shutdown(wait=False,cancel_futures=True) + # logger.info(results) + # for content in results: + # if content and content.get('content'): + + # item_dict = { + # "url":content.get('url'), + # "name":content.get('url'), + # "snippet":content.get('content'), + # "content": content.get('content'), + # "length":len(content.get('content')) + # } + # content_list.append(item_dict) + # logger.info("URL: {}".format(url)) + # logger.info("=================") + if len(results)== 0 : + content_list = conv_links + return content_list + except Exception as ex: + logger.error(ex) + raise ex + + + def new_async_client(_app): if "claude-3" in _app.ctx.model.lower(): return AsyncAnthropic( @@ -436,6 +526,12 @@ async def server_init(_app): query, _app.ctx.search1api_key, ) + elif _app.ctx.backend == "SEARXNG": + logger.info(os.getenv("SEARXNG_BASE_URL")) + _app.ctx.search_function = lambda query: search_with_searXNG( + query, + os.getenv("SEARXNG_BASE_URL"), + ) else: raise RuntimeError("Backend must be BING, GOOGLE, SERPER or SEARCHAPI or SEARCH1API.") _app.ctx.model = os.getenv("LLM_MODEL") @@ -503,7 +599,7 @@ async def get_related_questions(_app, query, contexts): response = await client.beta.tools.messages.create( model=_app.ctx.model, system=_more_questions_prompt, - max_tokens=4096, + max_tokens=1000, tools=tools, messages=[ {"role": "user", "content": query}, @@ -561,7 +657,7 @@ async def get_related_questions(_app, query, contexts): request_body = { "model": _app.ctx.model, "messages": messages, - "max_tokens": 4096, + "max_tokens": 1000, "tools": tools, "tool_choice": { "type": "function", @@ -907,4 +1003,4 @@ app.static("/", os.path.join(BASE_DIR, "ui/index.html"), name="ui") if __name__ == "__main__": port = int(os.getenv("PORT") or 8800) workers = int(os.getenv("WORKERS") or 1) - app.run(host="0.0.0.0", port=port, workers=workers, debug=False) \ No newline at end of file + app.run(host="0.0.0.0", port=port, workers=workers, debug=False)