diff --git a/README.md b/README.md index 3e4725d..c72a135 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ FreeAskInternet is a completely free, private and locally running search aggrega - 🈚️ Completely FREE (no need for any API keys) - 💻 Completely LOCAL (no GPU need, any computer can run ) -- 🔐 Completely PRIVATE (all thing runing locally) +- 🔐 Completely PRIVATE (all thing running locally) - 👻 Runs WITHOUT LLM Hardware (NO GPU NEED!) - 🤩 Using Free ChatGPT3.5 API (NO API keys need! Thx OpenAI) - 🚀 Fast and easy to deploy with Docker Compose @@ -41,7 +41,7 @@ cd ./FreeAskInternet docker-compose up -d ``` -🎉 You should now be able to open the web interface on http://localhost:3000. Nothing else is exposed by default. +🎉 You should now be able to open the web interface on http://localhost:3030. Nothing else is exposed by default.( For old web interface, accessing http://localhost:3000) ### How to update to latest @@ -51,6 +51,7 @@ cd ./FreeAskInternet git pull docker compose rm backend docker image rm nashsu/free_ask_internet +docker image rm nashsu/free_ask_internet_ui docker-compose up -d ``` diff --git a/docker-compose.yaml b/docker-compose.yaml index 4e5870a..bf4a6a8 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -2,10 +2,20 @@ services: backend: image: docker.io/nashsu/free_ask_internet:latest depends_on: - - freegpt35 + - llm-freegpt35 restart: on-failure + freeaskinternet-ui: + image: docker.io/nashsu/free_ask_internet_ui:latest + ports: + - "3030:80" + environment: + BACKEND_HOST: "backend:8000" + depends_on: + - backend + restart: always + chatgpt-next-web: image: yidadaa/chatgpt-next-web ports: @@ -16,13 +26,32 @@ services: BASE_URL: "http://backend:8000" CUSTOM_MODELS: "-all,+gpt-3.5-turbo" depends_on: - - freegpt35 + - llm-freegpt35 restart: always - freegpt35: + llm-freegpt35: image: missuo/freegpt35:latest restart: always + llm-kimi: + image: vinlic/kimi-free-api:latest + restart: always + environment: + - TZ=Asia/Shanghai + + llm-glm4: + image: vinlic/glm-free-api:latest + restart: always + environment: + - TZ=Asia/Shanghai + + + llm-qwen: + image: vinlic/qwen-free-api:latest + restart: always + environment: + - TZ=Asia/Shanghai + searxng: image: docker.io/searxng/searxng:latest volumes: diff --git a/free_ask_internet.py b/free_ask_internet.py index 7c78a37..62d9f29 100644 --- a/free_ask_internet.py +++ b/free_ask_internet.py @@ -94,38 +94,69 @@ def search_web_ref(query:str, debug=False): print("URL: {}".format(url)) print("=================") - return content_list + return conv_links,content_list except Exception as ex: raise ex -def gen_prompt(question,content_list, context_length_limit=11000,debug=False): +def gen_prompt(question,content_list, lang="zh-CN", context_length_limit=11000,debug=False): limit_len = (context_length_limit - 2000) if len(question) > limit_len: question = question[0:limit_len] ref_content = [ item.get("content") for item in content_list] + + answer_language = ' Simplified Chinese ' + if lang == "zh-CN": + answer_language = ' Simplified Chinese ' + if lang == "zh-TW": + answer_language = ' Traditional Chinese ' + if lang == "en-US": + answer_language = ' English ' + if len(ref_content) > 0: + if False: + prompts = ''' + 您是一位由 nash_su 开发的大型语言人工智能助手。您将被提供一个用户问题，并需要撰写一个清晰、简洁且准确的答案。提供了一组与问题相关的上下文，每个都以[[citation:x]]这样的编号开头，x代表一个数字。请在适当的情况下在句子末尾引用上下文。答案必须正确、精确，并以专家的中立和职业语气撰写。请将答案限制在2000个标记内。不要提供与问题无关的信息，也不要重复。如果给出的上下文信息不足，请在相关主题后写上“信息缺失：”。请按照引用编号[citation:x]的格式在答案中对应部分引用上下文。如果一句话源自多个上下文，请列出所有相关的引用编号，例如[citation:3][citation:5]，不要将引用集中在最后返回，而是在答案对应部分列出。除非是代码、特定的名称或引用编号，答案的语言应与问题相同。以下是上下文的内容集： + ''' + "\n\n" + "```" + ref_index = 1 - prompts = ''' - 您是一位由 nash_su 开发的基于搜索引擎返回内容的AI问答助手。您将被提供一个用户问题，并需要撰写一个清晰、简洁且准确的答案。答案必须正确、精确，并以专家的中立和职业语气撰写。请将答案限制在2000个标记内。不要提供与问题无关的信息，也不要重复。如果给出的上下文信息不足，请在相关主题后写上“信息缺失：”。除非是代码、特定的名称或引用编号，答案的语言应与问题相同。以下是上下文的内容集： - ''' + "\n\n" + "```" - ref_index = 1 + for ref_text in ref_content: + + prompts = prompts + "\n\n" + " [citation:{}] ".format(str(ref_index)) + ref_text + ref_index += 1 - for ref_text in ref_content: - - prompts = prompts + "\n\n" + ref_text - ref_index += 1 + if len(prompts) >= limit_len: + prompts = prompts[0:limit_len] + prompts = prompts + ''' + ``` + 记住，不要一字不差的重复上下文内容. 回答必须使用简体中文，如果回答很长，请尽量结构化、分段落总结。请按照引用编号[citation:x]的格式在答案中对应部分引用上下文。如果一句话源自多个上下文，请列出所有相关的引用编号，例如[citation:3][citation:5]，不要将引用集中在最后返回，而是在答案对应部分列出。下面是用户问题： + ''' + question + else: + prompts = ''' + You are a large language AI assistant develop by nash_su. You are given a user question, and please write clean, concise and accurate answer to the question. You will be given a set of related contexts to the question, each starting with a reference number like [[citation:x]], where x is a number. Please use the context and cite the context at the end of each sentence if applicable. + Your answer must be correct, accurate and written by an expert using an unbiased and professional tone. Please limit to 1024 tokens. Do not give any information that is not related to the question, and do not repeat. Say "information is missing on" followed by the related topic, if the given context do not provide sufficient information. - if len(prompts) >= limit_len: - prompts = prompts[0:limit_len] - prompts = prompts + ''' -``` -记住，不要一字不差的重复上下文内容. 回答必须使用简体中文，如果回答很长，请尽量结构化、分段落总结。 下面是用户问题： -''' + question + Please cite the contexts with the reference numbers, in the format [citation:x]. If a sentence comes from multiple contexts, please list all applicable citations, like [citation:3][citation:5]. Other than code and specific names and citations, your answer must be written in the same language as the question. + Here are the set of contexts: + ''' + "\n\n" + "```" + ref_index = 1 + + for ref_text in ref_content: + + prompts = prompts + "\n\n" + " [citation:{}] ".format(str(ref_index)) + ref_text + ref_index += 1 + + if len(prompts) >= limit_len: + prompts = prompts[0:limit_len] + prompts = prompts + ''' + ``` + Above is the reference contexts. Remember, don't repeat the context word for word. Answer in ''' + answer_language + '''. If the response is lengthy, structure it in paragraphs and summarize where possible. Cite the context using the format [citation:x] where x is the reference number. If a sentence originates from multiple contexts, list all relevant citation numbers, like [citation:3][citation:5]. Don't cluster the citations at the end but include them in the answer where they correspond. + Remember, don't blindly repeat the contexts verbatim. And here is the user question: + ''' + question else: @@ -137,14 +168,33 @@ def gen_prompt(question,content_list, context_length_limit=11000,debug=False): return prompts +def chat(prompt, model:str,llm_auth_token:str,llm_base_url:str,using_custom_llm=False,stream=True, debug=False): + openai.base_url = "http://127.0.0.1:3040/v1/" + + if model == "gpt3.5": + openai.base_url = "http://llm-freegpt35:3040/v1/" + + if model == "kimi": + openai.base_url = "http://llm-kimi:8000/v1/" + if model == "glm4": + openai.base_url = "http://llm-glm4:8000/v1/" + if model == "qwen": + openai.base_url = "http://llm-qwen:8000/v1/" + + + if llm_auth_token == '': + llm_auth_token = "CUSTOM" + + openai.api_key = llm_auth_token + + if using_custom_llm: + openai.base_url = llm_base_url + openai.api_key = "CUSTOM" + -def chat(prompt, stream=True, debug=False): - openai.base_url = "http://freegpt35:3040/v1/" - openai.api_key = "EMPTY" total_content = "" for chunk in openai.chat.completions.create( - model="gpt-3.5-turbo", - # model='Qwen1.5-1.8B-Chat', + model=model, messages=[{ "role": "user", "content": prompt diff --git a/server.py b/server.py index 0cd8a70..582a8f4 100644 --- a/server.py +++ b/server.py @@ -26,6 +26,7 @@ from fastapi.middleware.cors import CORSMiddleware from contextlib import asynccontextmanager from typing import Any, Dict, List, Literal, Optional, Union from sse_starlette.sse import ServerSentEvent, EventSourceResponse +from fastapi.responses import StreamingResponse app = FastAPI() @@ -63,6 +64,15 @@ class DeltaMessage(BaseModel): content: Optional[str] = None +class QueryRequest(BaseModel): + query:str + model: str + ask_type: Literal["search", "llm"] + llm_auth_token: Optional[str] = "CUSTOM" + llm_base_url: Optional[str] = "" + using_custom_llm:Optional[bool] = False + lang:Optional[str] = "zh-CN" + class ChatCompletionRequest(BaseModel): model: str messages: List[ChatMessage] @@ -91,8 +101,21 @@ class ChatCompletionResponse(BaseModel): ChatCompletionResponseStreamChoice]] created: Optional[int] = Field(default_factory=lambda: int(time.time())) +class SearchItem(BaseModel): + url: str + icon_url: str + site_name:str + snippet:str + title:str - +class SearchItemList(BaseModel): + search_items: List[SearchItem] = [] + +class SearchResp(BaseModel): + code:int + msg:str + data: List[SearchItem] = [] + @app.get("/v1/models", response_model=ModelList) async def list_models(): @@ -156,11 +179,84 @@ def predict(query: str, history: None, model_id: str): +@app.post("/api/search/get_search_refs", response_model=SearchResp) +async def get_search_refs(request: QueryRequest): + + global search_results + search_results = [] + search_item_list = [] + if request.ask_type == "search": + search_links,search_results = free_ask_internet.search_web_ref(request.query) + for search_item in search_links: + snippet = search_item.get("snippet") + url = search_item.get("url") + icon_url = search_item.get("icon_url") + site_name = search_item.get("site_name") + title = search_item.get("title") + + + si = SearchItem(snippet=snippet,url=url,icon_url=icon_url,site_name=site_name,title=title) + + search_item_list.append(si) + + resp = SearchResp(code=0,msg="success",data=search_item_list) + + return resp + +def generator(prompt:str, model:str, llm_auth_token:str,llm_base_url:str, using_custom_llm=False,is_failed=False): + if is_failed: + yield "搜索失败，没有返回结果" + else: + total_token = "" + for token in free_ask_internet.chat(prompt=prompt,model=model,llm_auth_token=llm_auth_token,llm_base_url=llm_base_url,using_custom_llm=using_custom_llm,stream=True): + total_token += token + yield token + +@app.post("/api/search/stream/{search_uuid}") +async def stream(search_uuid:str,request: QueryRequest): + global search_results + + if request.ask_type == "llm": + + answer_language = ' Simplified Chinese ' + if request.lang == "zh-CN": + answer_language = ' Simplified Chinese ' + if request.lang == "zh-TW": + answer_language = ' Traditional Chinese ' + if request.lang == "en-US": + answer_language = ' English ' + prompt = ' You are a large language AI assistant develop by nash_su. Answer user question in ' + answer_language + '. And here is the user question: ' + request.query + generate = generator(prompt,model=request.model,llm_auth_token=request.llm_auth_token, llm_base_url=request.llm_base_url, using_custom_llm=request.using_custom_llm) + else: + prompt = None + limit_count = 10 + + while limit_count > 0: + try: + if len(search_results) > 0: + prompt = free_ask_internet.gen_prompt(request.query,search_results,lang=request.lang,context_length_limit=8000) + break + else: + limit_count -= 1 + time.sleep(1) + except Exception as err: + limit_count -= 1 + time.sleep(1) + total_token = "" + if prompt: + generate = generator(prompt,model=request.model,llm_auth_token=request.llm_auth_token, llm_base_url=request.llm_base_url, using_custom_llm=request.using_custom_llm) + else: + generate = generator(prompt,model=request.model,llm_auth_token=request.llm_auth_token,llm_base_url=request.llm_base_url, using_custom_llm=request.using_custom_llm,is_failed=True) + + # return EventSourceResponse(generate, media_type="text/event-stream") + return StreamingResponse(generate, media_type="text/event-stream") + def main(): port = 8000 - + search_results = [] + uvicorn.run(app, host='0.0.0.0', port=port, workers=1)