Merge pull request #13 from lizhe2004/main

support searxng search server

README.md

@@ -31,6 +31,9 @@ To use the Bing Web Search API, please visit [this link](https://www.microsoft.c
 ### Google Search
 You have three options for Google Search: you can use the [SearchApi Google Search API](https://www.searchapi.io/) from SearchApi, [Serper Google Search API](https://www.serper.dev) from Serper, or opt for the [Programmable Search Engine](https://developers.google.com/custom-search) provided by Google.
 
+### SearXNG Search
+you can host your personal [SearXNG server](https://github.com/searxng/searxng), then you do not need pay for the search api. You just need provide the server address in `SEARXNG_BASE_URL`, with [aurora](https://github.com/aurora-develop/aurora),you can have a free private ai search engine. Be sure you enable the json format for the SearXNG server.
+
 ## Deployment
 ### Zeabur
 Just click on it
@@ -92,7 +95,7 @@ This project provides some additional configuration items set with environment v
 | `LLM_MODEL`      | Yes       | The model you want to use,support all chat models of openai, groq and claude. | `gpt-3.5-turbo-0125,mixtral-8x7b-32768,claude-3-haiku-20240307...`   
 | `RELATED_QUESTIONS`      | No       | Show the related questions. | `1`   
 | `NODE_ENV`      | No       | The environment required for deployment is necessary only during manual deployment. | `production`   
-| `BACKEND`      | Yes       | The search service you want. | `SEARCH1API,BING,GOOGLE,SERPER,SEARCHAPI`   
+| `BACKEND`      | Yes       | The search service you want. | `SEARCH1API,BING,GOOGLE,SERPER,SEARCHAPI,SEARXNG`   
 | `CHAT_HISTORY`      | No       | Continue to ask about the results | `1`   
 | `SEARCH1API_KEY`      | Yes       | If you choose SEARCH1API. | `xxx`   
 | `BING_SEARCH_V7_SUBSCRIPTION_KEY`      | No       | If you choose BING. | `xxx`   
@@ -100,8 +103,8 @@ This project provides some additional configuration items set with environment v
 | `GOOGLE_SEARCH_API_KEY`      | No       | If you choose GOOGLE. | `xxx`   
 | `SEARCHAPI_API_KEY`      | No       | If you choose SEARCHAPI. | `xxx`   
 | `SERPER_SEARCH_API_KEY`      | No       | If you choose SERPER. | `xxx`
-| `NEXT_PUBLIC_GOOGLE_ANALYTICS`      | No       | You can use Google Analytics to know how many users you have on your website. | MEASUREMENT ID,you can find on your google analytics account,like `G-XXXXXX`   
-
+| `NEXT_PUBLIC_GOOGLE_ANALYTICS`      | No       | You can use Google Analytics to know how many users you have on your website. | MEASUREMENT ID,you can find on your google analytics account,like `G-XXXXXX`
+| `SEARXNG_BASE_URL` | No       | the hosted serxng server address. it is required when the BACKEND is `SEARXNG` | `https://serxng.xxx.com/`
 
 
 

requirements.txt

@@ -5,4 +5,7 @@ anthropic
 loguru
 sanic
 sqlitedict
-python-dotenv
+python-dotenv
+tld==0.13
+tldextract==5.1.2
+trafilatura==1.8.1

search4all.py

@@ -11,6 +11,12 @@ import asyncio
 from anthropic import AsyncAnthropic
 from loguru import logger
 from dotenv import load_dotenv
+import urllib.parse
+import trafilatura
+from trafilatura import bare_extraction
+import tldextract
+from concurrent.futures import ThreadPoolExecutor
+from urllib.parse import urlparse
 load_dotenv()
 
 import sanic
@@ -378,6 +384,90 @@ def search_with_searchapi(query: str, subscription_key: str):
         return []
 
 
+def extract_url_content(url):
+    logger.info(url)
+    downloaded = trafilatura.fetch_url(url)
+    content =  trafilatura.extract(downloaded)
+
+    logger.info(url +"______"+  content)
+    return {"url":url, "content":content}
+
+
+
+def search_with_searXNG(query:str,url:str):
+ 
+    content_list = []
+
+    try:
+        safe_string = urllib.parse.quote_plus(":auto " + query)
+        response = requests.get(url+'?q=' + safe_string + '&category=general&format=json&engines=bing%2Cgoogle')
+        response.raise_for_status()
+        search_results = response.json()
+
+        logger.info("JSON Response:")
+        logger.info(search_results)
+        pedding_urls = []
+
+        conv_links = []
+
+        if search_results.get('results'):
+            for item in search_results.get('results')[0:9]:
+                name = item.get('title')
+                snippet = item.get('content')
+                url = item.get('url')
+                pedding_urls.append(url)
+
+                if url:
+                    url_parsed = urlparse(url)
+                    domain = url_parsed.netloc
+                    icon_url =  url_parsed.scheme + '://' + url_parsed.netloc + '/favicon.ico'
+                    site_name = tldextract.extract(url).domain
+
+                conv_links.append({
+                    'site_name':site_name,
+                    'icon_url':icon_url,
+                    'title':name,
+                    'name':name,
+                    'url':url,
+                    'snippet':snippet
+                })
+            logger.info(conv_links)
+            results = []
+            futures = []
+
+            # executor = ThreadPoolExecutor(max_workers=10) 
+            # for url in pedding_urls:
+            #     futures.append(executor.submit(extract_url_content,url))
+            # try:
+            #     for future in futures:
+            #         res = future.result(timeout=5)
+            #         results.append(res)
+            # except concurrent.futures.TimeoutError:
+            #     logger.error("任务执行超时")
+            #     executor.shutdown(wait=False,cancel_futures=True)
+            # logger.info(results)
+            # for content in results:
+            #     if content and content.get('content'):
+                    
+            #         item_dict = {
+            #             "url":content.get('url'),
+            #             "name":content.get('url'),
+            #             "snippet":content.get('content'),
+            #             "content": content.get('content'),
+            #             "length":len(content.get('content'))
+            #         }
+            #         content_list.append(item_dict)
+            #     logger.info("URL: {}".format(url))
+            #     logger.info("=================")
+        if len(results)== 0 :
+            content_list = conv_links
+        return  content_list
+    except Exception as ex:
+        logger.error(ex)
+        raise ex
+
+
+
 def new_async_client(_app):
     if "claude-3" in _app.ctx.model.lower():
         return AsyncAnthropic(
@@ -436,6 +526,12 @@ async def server_init(_app):
             query,
             _app.ctx.search1api_key,
         )
+    elif _app.ctx.backend == "SEARXNG":
+        logger.info(os.getenv("SEARXNG_BASE_URL"))
+        _app.ctx.search_function = lambda query: search_with_searXNG(
+            query, 
+            os.getenv("SEARXNG_BASE_URL"),
+        )
     else:
         raise RuntimeError("Backend must be BING, GOOGLE, SERPER or SEARCHAPI or SEARCH1API.")
     _app.ctx.model = os.getenv("LLM_MODEL")
@@ -503,7 +599,7 @@ async def get_related_questions(_app, query, contexts):
             response = await client.beta.tools.messages.create(
                 model=_app.ctx.model,
                 system=_more_questions_prompt,
-                max_tokens=4096,
+                max_tokens=1000,
                 tools=tools,  
                 messages=[
                 {"role": "user", "content": query},
@@ -561,7 +657,7 @@ async def get_related_questions(_app, query, contexts):
             request_body = {
                 "model": _app.ctx.model,
                 "messages": messages,
-                "max_tokens": 4096,
+                "max_tokens": 1000,
                 "tools": tools,
                 "tool_choice": {
                 "type": "function",
@@ -907,4 +1003,4 @@ app.static("/", os.path.join(BASE_DIR, "ui/index.html"), name="ui")
 if __name__ == "__main__":
     port = int(os.getenv("PORT") or 8800)
     workers = int(os.getenv("WORKERS") or 1)
-    app.run(host="0.0.0.0", port=port, workers=workers, debug=False)
+    app.run(host="0.0.0.0", port=port, workers=workers, debug=False)