![](\"./assets/LiveTalking-logo.jpg\"){kind=logo}
\n\n \n
\n
\n ![](\"https://img.shields.io/badge/license-Apache%202-dfd.svg\"){kind=icon}
\n ![](\"https://img.shields.io/github/v/release/lipku/LiveTalking?color=ffa\")
![](\"https://img.shields.io/badge/python-3.10+-aff.svg\"){kind=icon}
\n ![](\"https://img.shields.io/badge/os-linux%2C%20win%2C%20mac-pink.svg\"){kind=icon}
\n ![](\"https://img.shields.io/github/contributors/lipku/LiveTalking?color=c4f042&style=flat-square\")
![](\"https://img.shields.io/github/forks/lipku/LiveTalking?color=8ae8ff\")
![](\"https://img.shields.io/github/stars/lipku/LiveTalking?color=ccf\")
![](\"./assets/qrcode-wechat.jpg\")
"
},
{
"path": "README.md",
"content": " # [English](./README-EN.md) | 中文版 \n \n \n
\n
\n \n \n \n
\n\n"
},
{
"path": "app.py",
"content": "###############################################################################\n# Copyright (C) 2024 LiveTalking@lipku https://github.com/lipku/LiveTalking\n# email: lipku@foxmail.com\n# \n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n# \n# http://www.apache.org/licenses/LICENSE-2.0\n# \n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n###############################################################################\n\n# server.py\nfrom flask import Flask, render_template,send_from_directory,request, jsonify\nfrom flask_sockets import Sockets\nimport base64\nimport json\n#import gevent\n#from gevent import pywsgi\n#from geventwebsocket.handler import WebSocketHandler\nimport re\nimport numpy as np\nfrom threading import Thread,Event\n#import multiprocessing\nimport torch.multiprocessing as mp\n\nfrom aiohttp import web\nimport aiohttp\nimport aiohttp_cors\nfrom aiortc import RTCPeerConnection, RTCSessionDescription,RTCIceServer,RTCConfiguration\nfrom aiortc.rtcrtpsender import RTCRtpSender\nfrom webrtc import HumanPlayer\nfrom basereal import BaseReal\nfrom llm import llm_response\n\nimport argparse\nimport random\nimport shutil\nimport asyncio\nimport torch\nfrom typing import Dict\nfrom logger import logger\nimport gc\n\n\napp = Flask(__name__)\n#sockets = Sockets(app)\nnerfreals:Dict[int, BaseReal] = {} #sessionid:BaseReal\nopt = None\nmodel = None\navatar = None\n \n\n#####webrtc###############################\npcs = set()\n\ndef randN(N)->int:\n '''生成长度为 N的随机数 '''\n min = pow(10, N - 1)\n max = pow(10, N)\n return random.randint(min, max - 1)\n\ndef build_nerfreal(sessionid:int)->BaseReal:\n opt.sessionid=sessionid\n if opt.model == 'wav2lip':\n from lipreal import LipReal\n nerfreal = LipReal(opt,model,avatar)\n elif opt.model == 'musetalk':\n from musereal import MuseReal\n nerfreal = MuseReal(opt,model,avatar)\n # elif opt.model == 'ernerf':\n # from nerfreal import NeRFReal\n # nerfreal = NeRFReal(opt,model,avatar)\n elif opt.model == 'ultralight':\n from lightreal import LightReal\n nerfreal = LightReal(opt,model,avatar)\n return nerfreal\n\n#@app.route('/offer', methods=['POST'])\nasync def offer(request):\n params = await request.json()\n offer = RTCSessionDescription(sdp=params[\"sdp\"], type=params[\"type\"])\n\n # if len(nerfreals) >= opt.max_session:\n # logger.info('reach max session')\n # return web.Response(\n # content_type=\"application/json\",\n # text=json.dumps(\n # {\"code\": -1, \"msg\": \"reach max session\"}\n # ),\n # )\n sessionid = randN(6) #len(nerfreals)\n nerfreals[sessionid] = None\n logger.info('sessionid=%d, session num=%d',sessionid,len(nerfreals))\n nerfreal = await asyncio.get_event_loop().run_in_executor(None, build_nerfreal,sessionid)\n nerfreals[sessionid] = nerfreal\n \n #ice_server = RTCIceServer(urls='stun:stun.l.google.com:19302')\n ice_server = RTCIceServer(urls='stun:stun.freeswitch.org:3478')\n pc = RTCPeerConnection(configuration=RTCConfiguration(iceServers=[ice_server]))\n pcs.add(pc)\n\n @pc.on(\"connectionstatechange\")\n async def on_connectionstatechange():\n logger.info(\"Connection state is %s\" % pc.connectionState)\n if pc.connectionState == \"failed\":\n await pc.close()\n pcs.discard(pc)\n del nerfreals[sessionid]\n if pc.connectionState == \"closed\":\n pcs.discard(pc)\n del nerfreals[sessionid]\n # gc.collect()\n\n player = HumanPlayer(nerfreals[sessionid])\n audio_sender = pc.addTrack(player.audio)\n video_sender = pc.addTrack(player.video)\n capabilities = RTCRtpSender.getCapabilities(\"video\")\n preferences = list(filter(lambda x: x.name == \"H264\", capabilities.codecs))\n preferences += list(filter(lambda x: x.name == \"VP8\", capabilities.codecs))\n preferences += list(filter(lambda x: x.name == \"rtx\", capabilities.codecs))\n transceiver = pc.getTransceivers()[1]\n transceiver.setCodecPreferences(preferences)\n\n await pc.setRemoteDescription(offer)\n\n answer = await pc.createAnswer()\n await pc.setLocalDescription(answer)\n\n #return jsonify({\"sdp\": pc.localDescription.sdp, \"type\": pc.localDescription.type})\n\n return web.Response(\n content_type=\"application/json\",\n text=json.dumps(\n {\"sdp\": pc.localDescription.sdp, \"type\": pc.localDescription.type, \"sessionid\":sessionid}\n ),\n )\n\nasync def human(request):\n try:\n params = await request.json()\n\n sessionid = params.get('sessionid',0)\n if params.get('interrupt'):\n nerfreals[sessionid].flush_talk()\n\n if params['type']=='echo':\n nerfreals[sessionid].put_msg_txt(params['text'])\n elif params['type']=='chat':\n asyncio.get_event_loop().run_in_executor(None, llm_response, params['text'],nerfreals[sessionid]) \n #nerfreals[sessionid].put_msg_txt(res)\n\n return web.Response(\n content_type=\"application/json\",\n text=json.dumps(\n {\"code\": 0, \"msg\":\"ok\"}\n ),\n )\n except Exception as e:\n logger.exception('exception:')\n return web.Response(\n content_type=\"application/json\",\n text=json.dumps(\n {\"code\": -1, \"msg\": str(e)}\n ),\n )\n\nasync def interrupt_talk(request):\n try:\n params = await request.json()\n\n sessionid = params.get('sessionid',0)\n nerfreals[sessionid].flush_talk()\n \n return web.Response(\n content_type=\"application/json\",\n text=json.dumps(\n {\"code\": 0, \"msg\":\"ok\"}\n ),\n )\n except Exception as e:\n logger.exception('exception:')\n return web.Response(\n content_type=\"application/json\",\n text=json.dumps(\n {\"code\": -1, \"msg\": str(e)}\n ),\n )\n\nasync def humanaudio(request):\n try:\n form= await request.post()\n sessionid = int(form.get('sessionid',0))\n fileobj = form[\"file\"]\n filename=fileobj.filename\n filebytes=fileobj.file.read()\n nerfreals[sessionid].put_audio_file(filebytes)\n\n return web.Response(\n content_type=\"application/json\",\n text=json.dumps(\n {\"code\": 0, \"msg\":\"ok\"}\n ),\n )\n except Exception as e:\n logger.exception('exception:')\n return web.Response(\n content_type=\"application/json\",\n text=json.dumps(\n {\"code\": -1, \"msg\": str(e)}\n ),\n )\n\nasync def set_audiotype(request):\n try:\n params = await request.json()\n\n sessionid = params.get('sessionid',0) \n nerfreals[sessionid].set_custom_state(params['audiotype'],params['reinit'])\n\n return web.Response(\n content_type=\"application/json\",\n text=json.dumps(\n {\"code\": 0, \"msg\":\"ok\"}\n ),\n )\n except Exception as e:\n logger.exception('exception:')\n return web.Response(\n content_type=\"application/json\",\n text=json.dumps(\n {\"code\": -1, \"msg\": str(e)}\n ),\n )\n\nasync def record(request):\n try:\n params = await request.json()\n\n sessionid = params.get('sessionid',0)\n if params['type']=='start_record':\n # nerfreals[sessionid].put_msg_txt(params['text'])\n nerfreals[sessionid].start_recording()\n elif params['type']=='end_record':\n nerfreals[sessionid].stop_recording()\n return web.Response(\n content_type=\"application/json\",\n text=json.dumps(\n {\"code\": 0, \"msg\":\"ok\"}\n ),\n )\n except Exception as e:\n logger.exception('exception:')\n return web.Response(\n content_type=\"application/json\",\n text=json.dumps(\n {\"code\": -1, \"msg\": str(e)}\n ),\n )\n\nasync def is_speaking(request):\n params = await request.json()\n\n sessionid = params.get('sessionid',0)\n return web.Response(\n content_type=\"application/json\",\n text=json.dumps(\n {\"code\": 0, \"data\": nerfreals[sessionid].is_speaking()}\n ),\n )\n\n\nasync def on_shutdown(app):\n # close peer connections\n coros = [pc.close() for pc in pcs]\n await asyncio.gather(*coros)\n pcs.clear()\n\nasync def post(url,data):\n try:\n async with aiohttp.ClientSession() as session:\n async with session.post(url,data=data) as response:\n return await response.text()\n except aiohttp.ClientError as e:\n logger.info(f'Error: {e}')\n\nasync def run(push_url,sessionid):\n nerfreal = await asyncio.get_event_loop().run_in_executor(None, build_nerfreal,sessionid)\n nerfreals[sessionid] = nerfreal\n\n pc = RTCPeerConnection()\n pcs.add(pc)\n\n @pc.on(\"connectionstatechange\")\n async def on_connectionstatechange():\n logger.info(\"Connection state is %s\" % pc.connectionState)\n if pc.connectionState == \"failed\":\n await pc.close()\n pcs.discard(pc)\n\n player = HumanPlayer(nerfreals[sessionid])\n audio_sender = pc.addTrack(player.audio)\n video_sender = pc.addTrack(player.video)\n\n await pc.setLocalDescription(await pc.createOffer())\n answer = await post(push_url,pc.localDescription.sdp)\n await pc.setRemoteDescription(RTCSessionDescription(sdp=answer,type='answer'))\n##########################################\n# os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'\n# os.environ['MULTIPROCESSING_METHOD'] = 'forkserver' \nif __name__ == '__main__':\n mp.set_start_method('spawn')\n parser = argparse.ArgumentParser()\n \n # audio FPS\n parser.add_argument('--fps', type=int, default=50, help=\"audio fps,must be 50\")\n # sliding window left-middle-right length (unit: 20ms)\n parser.add_argument('-l', type=int, default=10)\n parser.add_argument('-m', type=int, default=8)\n parser.add_argument('-r', type=int, default=10)\n\n parser.add_argument('--W', type=int, default=450, help=\"GUI width\")\n parser.add_argument('--H', type=int, default=450, help=\"GUI height\")\n\n #musetalk opt\n parser.add_argument('--avatar_id', type=str, default='avator_1', help=\"define which avatar in data/avatars\")\n #parser.add_argument('--bbox_shift', type=int, default=5)\n parser.add_argument('--batch_size', type=int, default=16, help=\"infer batch\")\n\n parser.add_argument('--customvideo_config', type=str, default='', help=\"custom action json\")\n\n parser.add_argument('--tts', type=str, default='edgetts', help=\"tts service type\") #xtts gpt-sovits cosyvoice fishtts tencent doubao indextts2 azuretts\n parser.add_argument('--REF_FILE', type=str, default=\"zh-CN-YunxiaNeural\",help=\"参考文件名或语音模型ID，默认值为 edgetts的语音模型ID zh-CN-YunxiaNeural, 若--tts指定为azuretts, 可以使用Azure语音模型ID, 如zh-CN-XiaoxiaoMultilingualNeural\")\n parser.add_argument('--REF_TEXT', type=str, default=None)\n parser.add_argument('--TTS_SERVER', type=str, default='http://127.0.0.1:9880') # http://localhost:9000\n # parser.add_argument('--CHARACTER', type=str, default='test')\n # parser.add_argument('--EMOTION', type=str, default='default')\n\n parser.add_argument('--model', type=str, default='musetalk') #musetalk wav2lip ultralight\n\n parser.add_argument('--transport', type=str, default='rtcpush') #webrtc rtcpush virtualcam\n parser.add_argument('--push_url', type=str, default='http://localhost:1985/rtc/v1/whip/?app=live&stream=livestream') #rtmp://localhost/live/livestream\n\n parser.add_argument('--max_session', type=int, default=1) #multi session count\n parser.add_argument('--listenport', type=int, default=8010, help=\"web listen port\")\n\n opt = parser.parse_args()\n #app.config.from_object(opt)\n #print(app.config)\n opt.customopt = []\n if opt.customvideo_config!='':\n with open(opt.customvideo_config,'r') as file:\n opt.customopt = json.load(file)\n\n # if opt.model == 'ernerf': \n # from nerfreal import NeRFReal,load_model,load_avatar\n # model = load_model(opt)\n # avatar = load_avatar(opt) \n if opt.model == 'musetalk':\n from musereal import MuseReal,load_model,load_avatar,warm_up\n logger.info(opt)\n model = load_model()\n avatar = load_avatar(opt.avatar_id) \n warm_up(opt.batch_size,model) \n elif opt.model == 'wav2lip':\n from lipreal import LipReal,load_model,load_avatar,warm_up\n logger.info(opt)\n model = load_model(\"./models/wav2lip.pth\")\n avatar = load_avatar(opt.avatar_id)\n warm_up(opt.batch_size,model,256)\n elif opt.model == 'ultralight':\n from lightreal import LightReal,load_model,load_avatar,warm_up\n logger.info(opt)\n model = load_model(opt)\n avatar = load_avatar(opt.avatar_id)\n warm_up(opt.batch_size,avatar,160)\n\n # if opt.transport=='rtmp':\n # thread_quit = Event()\n # nerfreals[0] = build_nerfreal(0)\n # rendthrd = Thread(target=nerfreals[0].render,args=(thread_quit,))\n # rendthrd.start()\n if opt.transport=='virtualcam':\n thread_quit = Event()\n nerfreals[0] = build_nerfreal(0)\n rendthrd = Thread(target=nerfreals[0].render,args=(thread_quit,))\n rendthrd.start()\n\n #############################################################################\n appasync = web.Application(client_max_size=1024**2*100)\n appasync.on_shutdown.append(on_shutdown)\n appasync.router.add_post(\"/offer\", offer)\n appasync.router.add_post(\"/human\", human)\n appasync.router.add_post(\"/humanaudio\", humanaudio)\n appasync.router.add_post(\"/set_audiotype\", set_audiotype)\n appasync.router.add_post(\"/record\", record)\n appasync.router.add_post(\"/interrupt_talk\", interrupt_talk)\n appasync.router.add_post(\"/is_speaking\", is_speaking)\n appasync.router.add_static('/',path='web')\n\n # Configure default CORS settings.\n cors = aiohttp_cors.setup(appasync, defaults={\n \"*\": aiohttp_cors.ResourceOptions(\n allow_credentials=True,\n expose_headers=\"*\",\n allow_headers=\"*\",\n )\n })\n # Configure CORS on all routes.\n for route in list(appasync.router.routes()):\n cors.add(route)\n\n pagename='webrtcapi.html'\n if opt.transport=='rtmp':\n pagename='echoapi.html'\n elif opt.transport=='rtcpush':\n pagename='rtcpushapi.html'\n logger.info('start http server; http://