gavin/duix.ai
1
0
Fork 0
mirror of https://github.com/GuijiAI/duix.ai.git synced 2026-03-12 17:51:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

优化渲染线程，在音频计算嘴形和释放资源时添加锁

Browse Source
This commit is contained in:
jingtonghuai
2025-02-20 17:30:20 +08:00
parent b4d17df3ed
commit 39ea8fd55e
3 changed files with 280 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
duix-android/dh_aigc_android/duix-sdk/src/main/java/ai/guiji/duix/sdk/client/DUIX.java
View File

@@ -1,56 +1,31 @@
package ai.guiji.duix.sdk.client;
import android.content.Context;
import android.os.Handler;
import android.os.HandlerThread;
import com.btows.ncnntest.SCRFDNcnn;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import ai.guiji.duix.sdk.client.bean.ModelInfo;
import ai.guiji.duix.sdk.client.render.RenderSink;
import ai.guiji.duix.sdk.client.thread.DUIXThread;
import ai.guiji.duix.sdk.client.util.Logger;
import ai.guiji.duix.sdk.client.thread.RenderThread;
public class DUIX {
private final SCRFDNcnn scrfdncnn;
private final Context mContext;
private final Callback mCallback;
private final String baseDir;
private final String modelDir;
private final RenderSink renderSink;
private ExecutorService commonExecutor = Executors.newSingleThreadExecutor();
private final int sessionKey;
private final DUIXThread duixRender;
private final Thread renderThread;
private final Handler mHandler; // 处理心跳的异步线程
private RenderThread mRenderThread;
private boolean isReady; // 准备完成的标记
public DUIX(Context context, String baseDir, String modelDir, RenderSink sink, Callback callback) {
this.mContext = context;
this.mCallback = callback;
this.baseDir = baseDir;
this.modelDir = modelDir;
scrfdncnn = new SCRFDNcnn();
HandlerThread handlerThread = new HandlerThread("DUIX");
handlerThread.start();
mHandler = new Handler(handlerThread.getLooper());
sessionKey = (int) (System.currentTimeMillis() / 1000);
scrfdncnn.createdigit(sessionKey, new SCRFDNcnn.Callback() {
@Override
public void onMessageCallback(int what, int arg1, long arg2, String msg1, String msg2, Object object) {
Logger.d("onMessageCallback what" + what + " arg1: " + arg1 + " arg2: " + arg2 + " msg1: " + msg1 + " msg2: " + msg2);
}
});
duixRender = new DUIXThread(context, scrfdncnn, sink, mCallback);
renderThread = new Thread(duixRender);
renderThread.setName("DUIXRender-Thread");
renderThread.start();
this.renderSink = sink;
}
public boolean isReady() {
@@ -61,17 +36,62 @@ public class DUIX {
* 模型读取，需要异步操作
*/
public void init() {
if (commonExecutor != null) {
commonExecutor.execute(this::loadModel);
if (mRenderThread != null) {
mRenderThread.stopPreview();
mRenderThread = null;
}
mRenderThread = new RenderThread(mContext, baseDir, modelDir, renderSink, new RenderThread.RenderCallback() {
@Override
public void onInitResult(int code, int subCode, String message) {
isReady = true;
if (mCallback != null){
if (code == 0){
mCallback.onEvent(Constant.CALLBACK_EVENT_INIT_READY, "init ok", null);
} else {
mCallback.onEvent(Constant.CALLBACK_EVENT_INIT_ERROR, "init error code: " + subCode + " msg: " + message, null);
}
}
}
@Override
public void onPlayStart() {
if (mCallback != null){
mCallback.onEvent(Constant.CALLBACK_EVENT_AUDIO_PLAY_START, "play start", null);
}
}
@Override
public void onPlayEnd() {
if (mCallback != null){
mCallback.onEvent(Constant.CALLBACK_EVENT_AUDIO_PLAY_END, "play end", null);
}
}
@Override
public void onPlayProgress(long current, long total) {
float progress = current * 1.0F / total;
if (mCallback != null){
mCallback.onEvent(Constant.CALLBACK_EVENT_AUDIO_PLAY_PROGRESS, "audio play progress", progress);
}
}
@Override
public void onPlayError(int code, String msg) {
if (mCallback != null){
mCallback.onEvent(Constant.CALLBACK_EVENT_AUDIO_PLAY_ERROR, "audio play error code: " + code + " msg: " + msg, null);
}
}
});
mRenderThread.setName("DUIXRender-Thread");
mRenderThread.start();
}
/**
* 播放动作区间
*/
public void motion() {
if (isReady && duixRender != null) {
duixRender.requireMotion();
if (mRenderThread != null) {
mRenderThread.requireMotion();
}
}
@@ -81,8 +101,8 @@ public class DUIX {
* @param wavPath 16k采样率单通道16位深的wav本地文件
*/
public void playAudio(String wavPath) {
if (isReady && duixRender != null) {
duixRender.prepareAudio(wavPath);
if (isReady && mRenderThread != null) {
mRenderThread.prepareAudio(wavPath);
}
}
@@ -90,8 +110,8 @@ public class DUIX {
* 停止音频播放
*/
public boolean stopAudio() {
if (isReady && duixRender != null) {
duixRender.stopAudio();
if (isReady && mRenderThread != null) {
mRenderThread.stopAudio();
return true;
} else {
return false;
@@ -100,40 +120,12 @@ public class DUIX {
public void release() {
isReady = false;
if (mHandler != null) {
mHandler.removeCallbacksAndMessages(null);
mHandler.getLooper().quitSafely();
}
if (commonExecutor != null) {
commonExecutor.shutdown();
commonExecutor = null;
}
if (duixRender != null) {
duixRender.stopPreview();
}
if (renderThread != null) {
try {
renderThread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
scrfdncnn.stop();
scrfdncnn.reset();
scrfdncnn.releasedigit(sessionKey);
}
private void loadModel() {
ModelInfo info = ModelInfo.loadResource(scrfdncnn, baseDir, modelDir);
if (info != null) {
// 模型信息
duixRender.startPreview(info);
scrfdncnn.config(info.getNcnnConfig());
scrfdncnn.start();
isReady = true;
mCallback.onEvent(Constant.CALLBACK_EVENT_INIT_READY, "init ok", null);
} else {
mCallback.onEvent(Constant.CALLBACK_EVENT_INIT_ERROR, "init error, load model file error!", null);
if (mRenderThread != null) {
mRenderThread.stopPreview();
}
}
}

286
duix-android/dh_aigc_android/duix-sdk/src/main/java/ai/guiji/duix/sdk/client/thread/DUIXThread.java → duix-android/dh_aigc_android/duix-sdk/src/main/java/ai/guiji/duix/sdk/client/thread/RenderThread.java
View File

@@ -10,10 +10,12 @@ import android.util.Log;
import androidx.annotation.NonNull;
import com.btows.ncnntest.SCRFDNcnn;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.ExoPlaybackException;
import com.google.android.exoplayer2.MediaItem;
import com.google.android.exoplayer2.Player;
import com.google.android.exoplayer2.SimpleExoPlayer;
import com.google.android.exoplayer2.audio.AudioAttributes;
import com.google.android.exoplayer2.source.MediaSource;
import com.google.android.exoplayer2.source.ProgressiveMediaSource;
import com.google.android.exoplayer2.trackselection.AdaptiveTrackSelection;
@@ -40,21 +42,18 @@ import java.util.Random;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import ai.guiji.duix.sdk.client.Callback;
import ai.guiji.duix.sdk.client.Constant;
import ai.guiji.duix.sdk.client.bean.ImageFrame;
import ai.guiji.duix.sdk.client.bean.ModelInfo;
import ai.guiji.duix.sdk.client.render.RenderSink;
import ai.guiji.duix.sdk.client.util.Logger;
import ai.guiji.duix.sdk.client.util.MD5Util;
/**
* DUIX绘制线程，负责绘制线程及音频播放控制
*/
public class DUIXThread implements Runnable {
public class RenderThread extends Thread {
private static final int MSG_START_RENDER = 0; // 启动渲染
private static final int MSG_RENDER_STEP = 1; // 请求下一帧渲染
private static final int MSG_STOP_RENDER = 2; // 停止渲染
private static final int MSG_QUIT = 3; // 退出线程
@@ -63,16 +62,23 @@ public class DUIXThread implements Runnable {
private static final int MSG_PLAY_AUDIO = 6; // 音频的下载及ncnn计算完毕，准备播放
private static final int MSG_REQUIRE_MOTION = 7; // 请求播放动作区间
private boolean isRendering = false; // 为false时终止线程
private static final int MSG_PAUSE_AUDIO = 9; // 暂停播放音频
private static final int MSG_RESUME_AUDIO = 10; // 恢复播放音频
private volatile boolean isRendering = false; // 为false时终止线程
RenderHandler mHandler; // 使用该处理器来调度线程的事件
private final Object mReadyFence = new Object(); // 给isReady加一个对象锁
private boolean isReady; // handler等组件都创建完毕的标记
private final Object mBnfFence = new Object(); // 给isReady加一个对象锁
private final Context mContext;
private final SCRFDNcnn scrfdncnn;
private final Callback callback;
private SCRFDNcnn scrfdncnn;
private final RenderCallback callback;
private RenderSink mRenderSink;
private ExecutorService commonExecutor;
private ConcurrentLinkedQueue<ModelInfo.Frame> mPreviewQueue; // 播放帧
@@ -83,18 +89,24 @@ public class DUIXThread implements Runnable {
private ModelInfo mModelInfo; // 模型的全部信息都放在这里面
private ByteBuffer rawBuffer;
private ByteBuffer maskBuffer;
private final String baseConfigDir;
private final String modelDir;
private final RenderSink renderSink;
public DUIXThread(Context context, SCRFDNcnn scrfdncnn, RenderSink renderSink, Callback callback) {
public RenderThread(Context context, String baseConfigDir, String modelDir, RenderSink renderSink, RenderCallback callback) {
this.mContext = context;
this.scrfdncnn = scrfdncnn;
this.baseConfigDir = baseConfigDir;
this.modelDir = modelDir;
this.mRenderSink = renderSink;
this.callback = callback;
this.renderSink = renderSink;
}
public void setRenderSink(RenderSink renderSink){
this.mRenderSink = renderSink;
}
@Override
public void run() {
super.run();
Looper.prepare();
mHandler = new RenderHandler(this);
mPreviewQueue = new ConcurrentLinkedQueue<>();
@@ -107,6 +119,9 @@ public class DUIXThread implements Runnable {
.setTrackSelector(trackSelector)
.setBandwidthMeter(bandwidthMeter)
.build();
AudioAttributes attributes = new AudioAttributes.Builder().setUsage(C.USAGE_VOICE_COMMUNICATION).build();
mExoPlayer.setAudioAttributes(attributes, false);
mExoPlayer.setPlayWhenReady(true);
mExoPlayer.setRepeatMode(Player.REPEAT_MODE_OFF);
mExoPlayer.addListener(new Player.Listener() {
@@ -115,9 +130,13 @@ public class DUIXThread implements Runnable {
Player.Listener.super.onPlaybackStateChanged(state);
// Log.e("123", "onPlaybackStateChanged:" + state);
if (state == Player.STATE_READY) {
callback.onEvent(Constant.CALLBACK_EVENT_AUDIO_PLAY_START, "play start", null);
if (callback != null) {
callback.onPlayStart();
}
} else if (state == Player.STATE_ENDED) {
callback.onEvent(Constant.CALLBACK_EVENT_AUDIO_PLAY_END, "play end", null);
if (callback != null) {
callback.onPlayEnd();
}
}
}
@@ -125,36 +144,61 @@ public class DUIXThread implements Runnable {
public void onPlayerError(@NonNull ExoPlaybackException error) {
Player.Listener.super.onPlayerError(error);
Log.e("123", "onPlayerError:" + error);
callback.onEvent(Constant.CALLBACK_EVENT_AUDIO_PLAY_ERROR, "play error" + error.getMessage(), null);
if (callback != null) {
callback.onPlayError(-1000, "音频播放异常: " + error);
}
}
});
synchronized (mReadyFence) {
isReady = true;
mReadyFence.notify();
}
Looper.loop();
synchronized (mReadyFence) {
isReady = false;
mHandler = null;
}
}
public void startPreview(ModelInfo modelInfo) {
synchronized (mReadyFence) {
if (!isReady) {
try {
mReadyFence.wait();
} catch (InterruptedException e) {
e.printStackTrace();
scrfdncnn = new SCRFDNcnn();
int sessionKey = (int) (System.currentTimeMillis() / 1000);
scrfdncnn.createdigit(sessionKey, (SCRFDNcnn.Callback) (what, arg1, arg2, msg1, msg2, object) -> {
});
ModelInfo info = ModelInfo.loadResource(scrfdncnn, baseConfigDir, modelDir);
if (info != null) {
try {
scrfdncnn.config(info.getNcnnConfig());
scrfdncnn.start();
mModelInfo = info;
Logger.d("分辨率: " + mModelInfo.getWidth() + "x" + mModelInfo.getHeight());
rawBuffer = ByteBuffer.allocate(mModelInfo.getWidth() * mModelInfo.getHeight() * 3);
maskBuffer = ByteBuffer.allocate(mModelInfo.getWidth() * mModelInfo.getHeight() * 3);
if (!mModelInfo.isHasMask()) {
// 用纯白填充mask
Arrays.fill(maskBuffer.array(), (byte) 255);
}
Logger.d("模型初始化完成");
if (callback != null) {
callback.onInitResult(0, 0, mModelInfo.toString());
}
} catch (Exception e){
if (callback != null) {
callback.onInitResult(-1003, -1002, "模型加载异常: " + e);
}
}
if (mHandler != null) {
Message msg = new Message();
msg.what = MSG_START_RENDER;
msg.obj = modelInfo;
mHandler.sendMessage(msg);
} else {
if (callback != null) {
callback.onInitResult(-1003, -1001, "模型配置读取异常");
}
}
synchronized (mReadyFence) {
mReadyFence.notify();
}
isRendering = true;
handleAudioStep();
Looper.loop();
synchronized (mBnfFence) {
// 线程最后释放NCNN
scrfdncnn.stop();
scrfdncnn.reset();
scrfdncnn.releasedigit(sessionKey);
}
Logger.d("NCNN释放");
synchronized (mReadyFence) {
mHandler = null;
}
}
public void stopPreview() {
@@ -172,6 +216,18 @@ public class DUIXThread implements Runnable {
}
}
public void pauseAudio() {
if (mHandler != null) {
mHandler.sendEmptyMessage(MSG_PAUSE_AUDIO);
}
}
public void resumeAudio(){
if (mHandler != null) {
mHandler.sendEmptyMessage(MSG_RESUME_AUDIO);
}
}
public void stopAudio() {
if (mHandler != null) {
mHandler.sendEmptyMessage(MSG_STOP_AUDIO);
@@ -184,17 +240,8 @@ public class DUIXThread implements Runnable {
}
}
private void handleStartRender(ModelInfo modelInfo) {
isRendering = true;
mModelInfo = modelInfo;
Logger.d("分辨率: " + mModelInfo.getWidth() + "x" + mModelInfo.getHeight());
rawBuffer = ByteBuffer.allocate(mModelInfo.getWidth() * mModelInfo.getHeight() * 3);
maskBuffer = ByteBuffer.allocate(mModelInfo.getWidth() * mModelInfo.getHeight() * 3);
if (!mModelInfo.isHasMask()) {
// 用纯白填充mask
Arrays.fill(maskBuffer.array(), (byte) 255);
}
mHandler.sendEmptyMessage(MSG_RENDER_STEP);
public boolean isPlaying(){
return mExoPlayer != null && mExoPlayer.isPlaying();
}
private void handleAudioStep() {
@@ -212,7 +259,13 @@ public class DUIXThread implements Runnable {
}
} else {
if (commonExecutor != null) {
commonExecutor.shutdown();
commonExecutor.shutdownNow();
try {
boolean termination = commonExecutor.awaitTermination(300, TimeUnit.MILLISECONDS);
Logger.d("commonExecutor termination: " + termination);
} catch (InterruptedException e) {
Logger.e("中断commonExecutor异常: " + e);
}
}
if (mPreviewQueue != null) {
mPreviewQueue.clear();
@@ -255,8 +308,10 @@ public class DUIXThread implements Runnable {
if (frame != null) {
int audioBnf = -1;
if (mExoPlayer != null && mExoPlayer.isPlaying()) {
if (callback != null){
callback.onPlayProgress(mExoPlayer.getCurrentPosition(), mExoPlayer.getDuration());
}
float progress = mExoPlayer.getCurrentPosition() * 1.0F / mExoPlayer.getDuration();
callback.onEvent(Constant.CALLBACK_EVENT_AUDIO_PLAY_PROGRESS, "audio play progress", progress);
float curr = mTotalBnf * progress;
audioBnf = (int) curr;
}
@@ -275,8 +330,8 @@ public class DUIXThread implements Runnable {
int rst = scrfdncnn.drawonebuf(frame.rawPath, rawBuffer.array(), mModelInfo.getWidth() * mModelInfo.getHeight() * 3);
}
}
if (renderSink != null) {
renderSink.onVideoFrame(new ImageFrame(rawBuffer, maskBuffer, mModelInfo.getWidth(), mModelInfo.getHeight()));
if (mRenderSink != null) {
mRenderSink.onVideoFrame(new ImageFrame(rawBuffer, maskBuffer, mModelInfo.getWidth(), mModelInfo.getHeight()));
}
}
}
@@ -289,13 +344,73 @@ public class DUIXThread implements Runnable {
}
}
private void handlePrepareAudio(String wavPath) {
private void handlePrepareAudio(String path){
if (!isRendering){
return;
}
if (commonExecutor != null) {
commonExecutor.execute(() -> {
if (!TextUtils.isEmpty(wavPath)) {
loadAudio(wavPath);
} else {
callback.onEvent(Constant.CALLBACK_EVENT_AUDIO_PLAY_ERROR, "音频路径不能为空!", null);
String playPath = path;
if (!TextUtils.isEmpty(path)){
File wavFile = new File(path);
try (InputStream inputStream = new FileInputStream(wavFile)) {
byte[] headBuffer = new byte[44]; // 创建一个大小为44B的缓冲区
int ret = inputStream.read(headBuffer);
if (ret != -1) {
int chunkSize = headBuffer[4] + (headBuffer[5] << 8) + (headBuffer[6] << 16) + (headBuffer[7] << 24);
long fileLength = wavFile.length();
if (fileLength != chunkSize + 8) {
// wav头重写
int setChunkSize = (int) (fileLength - 8);
String setChunk2Id = "" + (char)headBuffer[36] + (char)headBuffer[37] + (char)headBuffer[38] + (char)headBuffer[39];
// Logger.d("setChunk2Id: " + setChunk2Id);
int setChunk2Size = (int) (fileLength - 44);
// Logger.d("setChunkSize: " + setChunkSize);
Logger.w("Wav头中的chunkSize和实际文件大小不一致chunkSize: " + chunkSize + " fileLength: " + fileLength + ", 尝试重写");
headBuffer[7] = (byte) (setChunkSize >> 24);
headBuffer[6] = (byte) ((setChunkSize << 8) >> 24);
headBuffer[5] = (byte) ((setChunkSize << 16) >> 24);
headBuffer[4] = (byte) ((setChunkSize << 24) >> 24);
if ("data".equals(setChunk2Id)){
headBuffer[43] = (byte) (setChunk2Size >> 24);
headBuffer[42] = (byte) ((setChunk2Size << 8) >> 24);
headBuffer[41] = (byte) ((setChunk2Size << 16) >> 24);
headBuffer[40] = (byte) ((setChunk2Size << 24) >> 24);
}
String modifyName = "modify_" + wavFile.getName();
File modifyWavFile = new File(wavFile.getParentFile(), modifyName);
OutputStream outStream = new BufferedOutputStream(new FileOutputStream(modifyWavFile));
outStream.write(headBuffer, 0, headBuffer.length);
byte[] buffer = new byte[1024 * 4];
while (inputStream.read(buffer) != -1) {
outStream.write(buffer);
}
outStream.flush();
outStream.close();
Logger.d("使用转换后的Wav文件: " + modifyWavFile.getAbsolutePath());
playPath = modifyWavFile.getAbsolutePath();
}
}
} catch (Exception e) {
Logger.e("音频文件头信息读取失败: " + e);
}
}
if (isRendering && !TextUtils.isEmpty(playPath)) {
synchronized (mBnfFence) {
long t1 = System.currentTimeMillis();
int all_bnf = scrfdncnn.onewav(playPath, "");
long t2 = System.currentTimeMillis();
Logger.d("all_bnf: " + all_bnf + " use: " + (t2-t1) + "(ms) text: " + playPath);
Message msg = new Message();
msg.what = MSG_PLAY_AUDIO;
msg.arg1 = all_bnf;
msg.obj = playPath;
if (mHandler != null) {
mHandler.sendMessage(msg);
}
}
}
});
}
@@ -310,8 +425,11 @@ public class DUIXThread implements Runnable {
}
private void handlePlayAudio(int all_bnf, String path) {
mTotalBnf = all_bnf;
Logger.d("收到所有嘴型信息 size: " + all_bnf);
if (all_bnf > 0){
mTotalBnf = all_bnf - 1;
} else {
mTotalBnf = all_bnf;
}
if (mExoPlayer != null) {
if (mExoPlayer.isPlaying()) {
mExoPlayer.stop();
@@ -326,16 +444,15 @@ public class DUIXThread implements Runnable {
}
}
private void loadAudio(String path) {
if (isRendering) {
int all_bnf = scrfdncnn.onewav(path, "");
Message msg = new Message();
msg.what = MSG_PLAY_AUDIO;
msg.arg1 = all_bnf;
msg.obj = path;
if (mHandler != null) {
mHandler.sendMessage(msg);
}
private void handlePauseAudio(){
if (mExoPlayer != null) {
mExoPlayer.pause();
}
}
private void handleResumeAudio(){
if (mExoPlayer != null) {
mExoPlayer.setPlayWhenReady(true);
}
}
@@ -345,23 +462,20 @@ public class DUIXThread implements Runnable {
static class RenderHandler extends Handler {
private final WeakReference<DUIXThread> encoderWeakReference;
private final WeakReference<RenderThread> encoderWeakReference;
public RenderHandler(DUIXThread render) {
public RenderHandler(RenderThread render) {
encoderWeakReference = new WeakReference<>(render);
}
@Override
public void handleMessage(Message msg) {
int what = msg.what;
DUIXThread render = encoderWeakReference.get();
RenderThread render = encoderWeakReference.get();
if (render == null) {
return;
}
switch (what) {
case MSG_START_RENDER:
render.handleStartRender((ModelInfo) msg.obj);
break;
case MSG_RENDER_STEP:
render.handleAudioStep();
break;
@@ -371,6 +485,12 @@ public class DUIXThread implements Runnable {
case MSG_PREPARE_AUDIO:
render.handlePrepareAudio((String) msg.obj);
break;
case MSG_PAUSE_AUDIO:
render.handlePauseAudio();
break;
case MSG_RESUME_AUDIO:
render.handleResumeAudio();
break;
case MSG_STOP_AUDIO:
render.handleStopAudio();
break;
@@ -391,4 +511,16 @@ public class DUIXThread implements Runnable {
}
}
public interface RenderCallback {
void onInitResult(int code, int subCode, String message);
void onPlayStart();
void onPlayEnd();
void onPlayProgress(long current, long total);
void onPlayError(int code, String msg);
}
}

12
duix-android/dh_aigc_android/test/src/main/java/ai/guiji/duix/test/ui/activity/CallActivity.kt
View File

@@ -186,10 +186,18 @@ class CallActivity : BaseActivity() {
input.close()
out.close()
File("${wavFile.absolutePath}.tmp").renameTo(wavFile)
duix?.playAudio(wavFile.absolutePath)
playAudioWithMotion(wavFile.absolutePath)
}
} else {
duix?.playAudio(wavFile.absolutePath)
playAudioWithMotion(wavFile.absolutePath)
}
}
private fun playAudioWithMotion(path: String){
runOnUiThread {
duix?.playAudio(path)
// 如果模型支持动作区间会播放动作区间
duix?.motion()
}
}