This commit is contained in:
wangjian
2024-05-16 17:22:43 +08:00
parent 36efbd7209
commit 6d54202241
4082 changed files with 999570 additions and 0 deletions

3
.idea/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,3 @@
# Default ignored files
/shelf/
/workspace.xml

12
.idea/duix.ai.iml generated Normal file
View File

@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Python 3.7 (py37)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="PyDocumentationSettings">
<option name="format" value="PLAIN" />
<option name="myDocStringFormat" value="Plain" />
</component>
</module>

View File

@@ -0,0 +1,22 @@
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<value>
<list size="9">
<item index="0" class="java.lang.String" itemvalue="scikit-image" />
<item index="1" class="java.lang.String" itemvalue="onnxruntime-gpu" />
<item index="2" class="java.lang.String" itemvalue="nvidia-pyindex" />
<item index="3" class="java.lang.String" itemvalue="fastapi" />
<item index="4" class="java.lang.String" itemvalue="opencv-python" />
<item index="5" class="java.lang.String" itemvalue="matplotlib" />
<item index="6" class="java.lang.String" itemvalue="grpcio" />
<item index="7" class="java.lang.String" itemvalue="uvicorn" />
<item index="8" class="java.lang.String" itemvalue="numexpr" />
</list>
</value>
</option>
</inspection_tool>
</profile>
</component>

View File

@@ -0,0 +1,6 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>

4
.idea/misc.xml generated Normal file
View File

@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (py37)" project-jdk-type="Python SDK" />
</project>

8
.idea/modules.xml generated Normal file
View File

@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/duix.ai.iml" filepath="$PROJECT_DIR$/.idea/duix.ai.iml" />
</modules>
</component>
</project>

6
.idea/vcs.xml generated Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

32
LICENSE Normal file
View File

@@ -0,0 +1,32 @@
Silicon Intelligence COMMUNITY LICENSE AGREEMENT
“Agreement” means the terms and conditions for use, reproduction, distribution and modification of this product forth herein.
“Documentation” means the specifications, manuals and documentation by Silicon Intelligence.
“Licensee” or “you” means you, or your employer or any other person or entity (if you are entering into this Agreement on such person or entitys behalf), of the age required under applicable laws, rules or regulations to provide legal consent and that has legal authority to bind your employer or such other person or entity if you are entering in this Agreement on their behalf.
“Silicon Intelligence Materials” means, collectively, Silicon Intelligences proprietary code and Documentation (and any portion thereof) made available under this Agreement.
By clicking “I Accept” below or by using or distributing any portion or element of the Silicon Intelligence Materials, you agree to be bound by this Agreement.
1. License Rights and Redistribution.
a. Grant of Rights. You are granted a non-exclusive, worldwide, non-transferable and royalty-free limited license under s intellectual property or other rights owned by Silicon Intelligence embodied in the SILICON INTELLIGENCE Materials to use, reproduce, distribute, copy, create derivative works of, and make modifications to the Silicon Intelligence Materials.
b. Redistribution and Use.
i. If you distribute or make available the Silicon Intelligence Materials (or any derivative works thereof), or a product or service that uses any of them, you shall (A) provide a copy of this Agreement with any such Silicon Intelligence Materials; and (B) prominently display “Built with Silicon Intelligence” on a related website, user interface, blogpost, about page, or product documentation. If you use the Silicon Intelligence Materials to create, train, fine tune, or otherwise improve an AI model, which is distributed or made available, you shall also include “Silicon Intelligence” at the beginning of any such AI model name.
ii. If you receive Silicon Intelligence Materials, or any derivative works thereof, from a Licensee as part of an integrated end user product, then Section 2 of this Agreement will not apply to you.
iii. You must retain in all copies of the Silicon Intelligence Materials that you distribute the following attribution notice within a “Notice” text file distributed as a part of such copies: “Silicon Intelligence is licensed under the Silicon Intelligence Community License, Copyright © Silicon Intelligence Platforms, Inc. All Rights Reserved.”
iv. Your use of the Silicon Intelligence Materials must comply with applicable laws and regulations (including trade compliance laws and regulations) .
2. Additional Commercial Terms. If, on the Silicon Intelligence duix.ai version release date, the monthly active users of the products or services made available by or for Licensee, or Licensees affiliates, is greater than 1 thousand monthly active users in the preceding calendar month, or your product based Silicon Intelligence material your active users greater 1 thousand, you must request a license from Silicon Intelligence, which Silicon Intelligence may grant to you in its sole discretion, and you are not authorized to exercise any of the rights under this Agreement unless or until Silicon Intelligence otherwise expressly grants you such rights.
3. Disclaimer of Warranty. UNLESS REQUIRED BY APPLICABLE LAW, THE SILICON INTELLIGENCE MATERIALS AND ANY OUTPUT AND RESULTS THEREFROM ARE PROVIDED ON AN “AS IS” BASIS, WITHOUT WARRANTIES OF ANY KIND, AND SILICON INTELLIGENCE DISCLAIMS ALL WARRANTIES OF ANY KIND, BOTH EXPRESS AND IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. YOU ARE SOLELY RESPONSIBLE FOR DETERMINING THE APPROPRIATENESS OF USING OR REDISTRIBUTING THE SILICON INTELLIGENCE MATERIALS AND ASSUME ANY RISKS ASSOCIATED WITH YOUR USE OF THE SILICON INTELLIGENCE MATERIALS AND ANY OUTPUT AND RESULTS.
4. Limitation of Liability. IN NO EVENT WILL SILICON INTELLIGENCE OR ITS AFFILIATES BE LIABLE UNDER ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, TORT, NEGLIGENCE, PRODUCTS LIABILITY, OR OTHERWISE, ARISING OUT OF THIS AGREEMENT, FOR ANY LOST PROFITS OR ANY INDIRECT, SPECIAL, CONSEQUENTIAL, INCIDENTAL, EXEMPLARY OR PUNITIVE DAMAGES, EVEN IF SILICON INTELLIGENCE OR ITS AFFILIATES HAVE BEEN ADVISED OF THE POSSIBILITY OF ANY OF THE FOREGOING.
5. Intellectual Property.
a. No trademark licenses are granted under this Agreement, and in connection with the Silicon Intelligence Materials, neither Silicon Intelligence nor Licensee may use any name or mark owned by or associated with the other or any of its affiliates, except as required for reasonable and customary use in describing and redistributing the Silicon Intelligence Materials or as set forth in this Section 5(a). Silicon Intelligence hereby grants you a license to use “Silicon Intelligence” solely as required to comply with the last sentence of Section 1.b.i. You will comply with Silicon Intelligences brand guidelines . All goodwill arising out of your use of the Mark will inure to the benefit of Silicon Intelligence.
b. If you institute litigation or other proceedings against Silicon Intelligenceor any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Silicon Intelligence Materials or outputs or results, or any portion of any of the foregoing, constitutes infringement of intellectual property or other rights owned or licensable by you, then any licenses granted to you under this Agreement shall terminate as of the date such litigation or claim is filed or instituted. You will indemnify and hold harmless Silicon Intelligence from and against any claim by any third party arising out of or related to your use or distribution of the Silicon Intelligence Materials.
6. Term and Termination. The term of this Agreement will commence upon your acceptance of this Agreement or access to the Silicon Intelligence Materials and will continue in full force and effect until terminated in accordance with the terms and conditions herein. Silicon Intelligence may terminate this Agreement if you are in breach of any term or condition of this Agreement. Upon termination of this Agreement, you shall delete and cease use of the Silicon Intelligence Materials. Sections 3, 4 shall survive the termination of this Agreement.

25
duix-android/dh_aigc_android/.gitignore vendored Normal file
View File

@@ -0,0 +1,25 @@
*.iml
.gradle
/local.properties
/.idea/
.DS_Store
/build/
*/build/
*/*/build/
*/debug/
*/release/
/captures
.externalNativeBuild
.cxx
*.bat
*.apk
output-metadata.json
# app用到zip的请忽略
# 自定义了local.properties的请删除这条
local.properties
~$*
gradlew
.idea
# 自定义了根目录gradle/gradle-wrapper.properties的请删除这条不推荐自定义
#gradle
build

View File

@@ -0,0 +1,279 @@
# 硅基数字人SDK
## 一、产品介绍
2D 数字人虚拟人SDK ,可以通过语音完成对虚拟人实时驱动。
### 1. 适用场景
部署成本低: 无需客户提供技术团队进行配合,支持低成本快速部署在多种终端及大屏;
网络依赖小:可落地在地铁、银行、政务等多种场景的虚拟助理自助服务上;
功能多样化:可根据客户需求满足视频、媒体、客服、金融、广电等多个行业的多样化需求
### 2. 核心功能
提供定制形象的 AI 主播,智能客服等多场景形象租赁,支持客户快速部署和低成本运营;
专属形象定制:支持定制专属的虚拟助理形象,可选低成本或深度形象生成;
播报内容定制:支持定制专属的播报内容,应用在培训、播报等多种场景上;
实时互动问答:支持实时对话,也可定制专属问答库,可满足咨询查询、语音闲聊、虚拟陪伴、垂类场景的客服问答等需求。
## 二、SDK集成
### 1. 支持的系统和硬件版本
| 项目 | 描述 |
|--------|------------------------------------------------------------------|
| 系统 | 支持 Android 7.0+ ( API Level 24 )到 Android 13 ( API Level 33 )系统。 |
| CPU架构 | armeabi-v7a, arm64-v8a |
| 硬件要求 | 要求设备 CPU4 核极以上,内存 4G 及以上。可用存储空间 500MB 及以上。 |
| 网络 | 支持 WIF 及移动网络。如果使用云端问答库,设备带宽(用于数字人的实际带宽)期望 10mbps 及以上。 |
| 开发 IDE | Android Studio Giraffe \mid 2022.3.1 Patch 2 |
| 内存要求 | 可用于数字人的内存 >= 400MB |
### 2. SDK集成
引入 sdk aar 包: duix_client_sdk_release_${version}.aar
app 目录新建 libs 目录,放入 aar 包,在 build.gradle 中增加配置如下
```gradle
dependencies {
// 使用aar包或直接引用SDK项目
implementation project(":duix-sdk")
// implementation fileTree(include: ['*.jar', '*.aar'], dir: 'libs')
// sdk 中使用到 exoplayer 处理音频(必选)
implementation 'com.google.android.exoplayer:exoplayer:2.14.2'
// 云端问答接口使用的SSE组件(非必选)
implementation 'com.squareup.okhttp3:okhttp-sse:4.10.0'
...
}
```
权限要求, AndroidManifest.xml中,增加如下配置
```xml
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.CHANGE_WIFI_STATE" />
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
<uses-permission android:name="android.permission.ACCESS_WIFI_STATE" />
<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" />
<uses-permission android:name="android.permission.WAKE_LOCK" />
</manifest>
```
## 三、SDK调用及API说明
### 1. 初始化SDK
在渲染页onCreate()阶段构建DUIX对象并添加回调事件
```kotlin
duix = DUIX(mContext, baseDir, modelDir, mDUIXRender) { event, msg, info ->
when (event) {
ai.guiji.duix.sdk.client.Constant.CALLBACK_EVENT_INIT_READY -> {
initOK()
}
ai.guiji.duix.sdk.client.Constant.CALLBACK_EVENT_INIT_ERROR -> {
}
// ...
}
}
// 异步回调结果
duix?.init()
```
DUIX对象构建说明:
| 参数 | 类型 | 描述 |
|----------|------------|-------------------------------------------|
| context | Context | 系统上下文 |
| baseDir | String | 存放模型驱动的配置文件,需要自行管理. 可将压缩文件解压到外部存储并提供文件夹路径 |
| modelDir | String | 存放模型文件的文件夹,需要自行管理. 可将压缩文件解压到外部存储并提供文件夹路径 |
| render | RenderSink | 渲染数据接口sdk提供了默认的渲染组件继承自该接口也可以自己实现 |
| callback | Callback | SDK处理的各种回调事件 |
参考demo LiveActivity示例
### 2. 获取SDK模型初始化状态
```kotlin
object : Callback {
fun onEvent(event: String, msg: String, info: Object) {
when (event) {
"init.ready" -> {
// SDK模型初始化成功
}
"init.error" -> {
//初始化失败
Log.e(TAG, "init error: $msg")
}
// ...
}
}
}
```
### 3. 数字人形象展示
使用RenderSink接口接受渲染帧数据SDK中提供了该接口实现DUIXRenderer.java。也可以自己实现该接口自定义渲染。
其中RenderSink的定义如下:
```java
/**
* 渲染管道,通过该接口返回渲染数据
*/
public interface RenderSink {
// frame中的buffer数据以bgr顺序排列
void onVideoFrame(ImageFrame imageFrame);
}
```
使用DUIXRenderer及DUIXTextureView控件简单实现渲染展示,该控件支持透明通道可以自由设置背景及前景:
```kotlin
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
// ...
mDUIXRender =
DUIXRenderer(
mContext,
binding.glTextureView
)
binding.glTextureView.setEGLContextClientVersion(GL_CONTEXT_VERSION)
binding.glTextureView.setEGLConfigChooser(8, 8, 8, 8, 16, 0) // 透明
binding.glTextureView.isOpaque = false // 透明
binding.glTextureView.setRenderer(mDUIXRender)
binding.glTextureView.renderMode =
GLSurfaceView.RENDERMODE_WHEN_DIRTY // 一定要在设置完Render之后再调用
duix = DUIX(mContext, duixOptions, mDUIXRender) { event, msg, _ ->
}
// ...
}
```
### 4. 启动数字人播报
在初始化成功后,可以播放音频以驱动形象
```kotlin
duix?.playAudio(wavPath)
```
参数说明:
| 参数 | 类型 | 描述 |
|---------|--------|-----------------------------|
| wavPath | String | 16k采样率单通道的wav文件地址或https网络地址 |
音频播放状态及进度回调:
```kotlin
object : Callback {
fun onEvent(event: String, msg: String, info: Object) {
when (event) {
// ...
"play.start" -> {
// 开始播放音频
}
"play.end" -> {
// 完成播放音频
}
"play.error" -> {
// 音频播放异常
}
"play.progress" -> {
// 音频播放进度
}
}
}
}
```
### 5. 终止当前播报
当数字人正在播报时调用该接口终止播报。
函数定义:
```
boolean stopAudio();
```
调用示例如下:
```kotlin
duix?.stopAudio()
```
### 6. 播放动作区间
当模型中支持播放动作区间时可使用该函数播放多做区间,多个时随机播放。
函数定义:
```
void motion();
```
调用示例如下:
```kotlin
duix?.motion()
```
###
### 四. Proguard配置
如果代码使用了混淆请在proguard-rules.pro中配置
```
-keep class com.btows.ncnntest.**{*; }
-dontwarn com.squareup.okhttp3.**
-keep class com.squareup.okhttp3.** { *;}
```
## 五、注意事项
1. 驱动渲染必须正确的配置基础配置文件夹和对应模型文件夹的存储路径。
2. 播放的音频文件不宜过大过大的音频导入会导致大量消耗CPU从而造成绘制卡顿。
## 六、版本记录
**<a>3.0.4</a>**
```text
1. 修复部分设备gl默认float低精度导致无法正常显示形象问题。
```
**<a>3.0.3</a>**
```text
1. 优化本地渲染。
```
## 七、其他相关的第三方开源项目
| 模块 | 描述 |
|-----------|--|
| [ExoPlayer](https://github.com/google/ExoPlayer) | 媒体播放器 |
| [okhttp](https://github.com/square/okhttp) | 网络框架 |
| [onnx](https://github.com/onnx/onnx) | 人工智能框架 |
| [ncnn](https://github.com/Tencent/ncnn) | 高性能神经网络计算框架 |

View File

@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="utf-8" ?>
<!-- https://github.com/bumptech/glide/issues/4940 -->
<lint>
<issue id="NotificationPermission">
<ignore regexp="com.bumptech.glide.request.target.NotificationTarget" />
</issue>
</lint>

View File

@@ -0,0 +1,39 @@
// Top-level build file where you can add configuration options common to all sub-projects/modules.
buildscript {
repositories {
maven { url 'https://maven.aliyun.com/repository/public/' }
maven { url 'https://maven.aliyun.com/repository/central' }
maven { url 'https://maven.aliyun.com/repository/google' }
maven { url 'https://maven.aliyun.com/repository/gradle-plugin' }
maven { url 'https://jitpack.io' }
maven { url 'https://repo1.maven.org/maven2/' }
google()
}
dependencies {
classpath 'com.android.tools.build:gradle:8.1.2'
classpath 'org.jetbrains.kotlin:kotlin-gradle-plugin:1.8.10'
}
}
allprojects {
repositories {
maven { url 'https://maven.aliyun.com/repository/public/' }
maven { url 'https://maven.aliyun.com/repository/central' }
maven { url 'https://maven.aliyun.com/repository/google' }
maven { url 'https://maven.aliyun.com/repository/gradle-plugin' }
maven { url 'https://jitpack.io' }
maven { url 'https://repo1.maven.org/maven2/' }
google()
}
}
ext {
compileSdkVersion = 33
buildToolsVersion = '30.0.2'
minSdkVersion = 24
targetSdkVersion = 33
versionCode = 2
versionName = "0.0.2"
}

Binary file not shown.

View File

@@ -0,0 +1 @@
/build

View File

@@ -0,0 +1,68 @@
plugins {
id 'com.android.library'
}
android {
namespace 'ai.guiji.duix.sdk.client'
compileSdk 33
defaultConfig {
minSdk 24
versionCode 4
versionName '3.0.3'
externalNativeBuild {
cmake {
abiFilters 'arm64-v8a', "armeabi-v7a"
cppFlags "-std=c++17", "-fexceptions"
//arguments "-DANDROID_STL=c++_shared","-DANDROID_TOOLCHAIN=clang"
}
}
}
buildTypes {
debug {
minifyEnabled false
proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
buildConfigField("String", "VERSION_NAME", "\"${defaultConfig.versionName}\"")
buildConfigField('int', 'VERSION_CODE', "${defaultConfig.versionCode}")
}
release {
minifyEnabled false
proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
buildConfigField("String", "VERSION_NAME", "\"${defaultConfig.versionName}\"")
buildConfigField('int', 'VERSION_CODE', "${defaultConfig.versionCode}")
android.libraryVariants.all { variant ->
variant.outputs.all {
outputFileName = "duix_client_sdk_${buildType.name}_${defaultConfig.versionName}.aar"
}
}
}
}
externalNativeBuild {
cmake {
path "src/main/cpp/CMakeLists.txt"
version "3.10.2"
}
}
compileOptions {
sourceCompatibility JavaVersion.VERSION_1_8
targetCompatibility JavaVersion.VERSION_1_8
}
// kotlinOptions {
// jvmTarget = '1.8'
// }
}
dependencies {
implementation 'com.google.android.exoplayer:exoplayer:2.14.2'
implementation "org.java-websocket:Java-WebSocket:1.5.1"
implementation 'com.squareup.okhttp3:okhttp-sse:4.10.0'
}

View File

@@ -0,0 +1,104 @@
# Add project specific ProGuard rules here.
# You can control the set of applied configuration files using the
# proguardFiles setting in build.gradle.
#
# For more details, see
# http://developer.android.com/guide/developing/tools/proguard.html
# If your project uses WebView with JS, uncomment the following
# and specify the fully qualified class name to the JavaScript interface
# class:
#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
# public *;
#}
# Uncomment this to preserve the line number information for
# debugging stack traces.
#-keepattributes SourceFile,LineNumberTable
# If you keep the line number information, uncomment this to
# hide the original source file name.
#-renamesourcefileattribute SourceFile
-optimizationpasses 5 #指定代码的压缩级别 0 - 7一般都是5无需改变
-dontusemixedcaseclassnames #不使用大小写混合
#告诉Proguard 不要跳过对非公开类的处理,默认是跳过
-dontskipnonpubliclibraryclasses #如果应用程序引入的有jar包并且混淆jar包里面的class
-verbose #混淆时记录日志(混淆后生产映射文件 map 类名 -> 转化后类名的映射
#指定混淆时的算法,后面的参数是一个过滤器
#这个过滤器是谷歌推荐的算法,一般也不会改变
-optimizations !code/simplification/arithmetic,!field/*,!class/merging/*
#类型转换错误 添加如下代码以便过滤泛型(不写可能会出现类型转换错误,一般情况把这个加上就是了),即避免泛型被混淆
-keepattributes Signature
#假如项目中有用到注解,应加入这行配置,对JSON实体映射也很重要,eg:fastjson
-keepattributes *Annotation*
#抛出异常时保留代码行数
-keepattributes SourceFile,LineNumberTable
#保持 native 的方法不去混淆
-keepclasseswithmembernames class * {
native <methods>;
}
#保持指定规则的方法不被混淆Android layout 布局文件中为控件配置的onClick方法不能混淆
-keepclassmembers class * extends android.app.Activity {
public void *(android.view.View);
}
#保持自定义控件指定规则的方法不被混淆
-keep public class * extends android.view.View {
public <init>(android.content.Context);
public <init>(android.content.Context, android.util.AttributeSet);
public <init>(android.content.Context, android.util.AttributeSet, int);
public void set*(...);
}
#保持枚举 enum 不被混淆
-keepclassmembers enum * {
public static **[] values();
public static ** valueOf(java.lang.String);
}
#保持 Parcelable 不被混淆aidl文件不能去混淆
-keep class * implements android.os.Parcelable {
public static final android.os.Parcelable$Creator *;
}
#需要序列化和反序列化的类不能被混淆Java反射用到的类也不能被混淆
-keepnames class * implements java.io.Serializable
#保护实现接口Serializable的类中指定规则的类成员不被混淆
-keepclassmembers class * implements java.io.Serializable {
static final long serialVersionUID;
private static final java.io.ObjectStreamField[] serialPersistentFields;
!static !transient <fields>;
private void writeObject(java.io.ObjectOutputStream);
private void readObject(java.io.ObjectInputStream);
java.lang.Object writeReplace();
java.lang.Object readResolve();
}
#保持R文件不被混淆否则你的反射是获取不到资源id的
-keep class **.R$* { *; }
-keepclassmembers class * {
public <init> (org.json.JSONObject);
}
-keepclassmembers enum * {
public static **[] values();
public static ** valueOf(java.lang.String);
}
#以下针对App本身设置
-keep class com.btows.ncnntest.**{*; }
-keep class ai.guiji.duix.sdk.client.render.** {*;}
-keep class ai.guiji.duix.sdk.client.render.**$* {*;}
-keep class ai.guiji.duix.sdk.client.bean.** {*;}
-keep class ai.guiji.duix.sdk.client.DUIX{*; }
-keep class ai.guiji.duix.sdk.client.DUIX$* {*;}
-keep class ai.guiji.duix.sdk.client.Constant{*; }
-keep class ai.guiji.duix.sdk.client.Constant* {*;}
-keep class ai.guiji.duix.sdk.client.DUIXOptions{*; }
-keep class ai.guiji.duix.sdk.client.DUIXOptions* {*;}
-keep class ai.guiji.duix.sdk.client.Callback{*; }
-keep class ai.guiji.duix.sdk.client.Callback* {*;}
-keep class ai.guiji.duix.sdk.client.render.DUIXTextureView{*; }
-keep class ai.guiji.duix.sdk.client.render.DUIXTextureView$* {*;}

View File

@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<uses-permission android:name="android.permission.INTERNET" />
</manifest>

View File

@@ -0,0 +1,86 @@
#/****************************************************************************
#* Cartoonifier, for Android.
#*****************************************************************************
#* by Shervin Emami, 5th Dec 2012 (shervin.emami@gmail.com)
#* http://www.shervinemami.info/
#*****************************************************************************
#* Ch1 of the book "Mastering OpenCV with Practical Computer Vision Projects"
#* Copyright Packt Publishing 2012.
#* http://www.packtpub.com/cool-projects-with-opencv/book
#****************************************************************************/
LOCAL_PATH := $(call my-dir)
include $(CLEAR_VARS)
LOCAL_SRC_FILES += \
base/coffeecatch.c \
android/DigitJni.cpp \
android/MsgcbJni.cpp \
android/JniHelper.cpp \
aisdk/jmat.cpp \
android/kmatarm.cpp \
aisdk/wavreader.cpp \
aisdk/wenet.cpp \
aisdk/aimodel.cpp \
aisdk/scrfd.cpp \
aisdk/pfpld.cpp \
aisdk/munet.cpp \
aisdk/blendgram.cpp \
aisdk/face_utils.cpp \
digit/netwav.cpp \
digit/looper.cpp \
digit/netcurl.cpp \
digit/GRender.cpp \
digit/GDigit.cpp \
digit/dispatchqueue.cpp \
render/EglRenderer.cpp \
render/RgbVideoRenderer.cpp \
render/SurfaceVideoRenderer.cpp \
render/RenderHelper.cpp \
render/AudioTrack.cpp \
render/AudioRenderer.cpp \
render/GlesProgram.cpp \
base/Log.cpp \
base/FrameSource.cpp \
base/MediaData.cpp \
base/MessageSource.cpp \
base/MessageHelper.cpp \
base/LoopThread.cpp \
base/XThread.cpp \
base/XTick.c \
base/cJSON.c \
base/dh_mem.c \
digit/grtcfg.c \
base/LoopThreadHelper.cpp \
)
LOCAL_ARM_NEON := true
LOCAL_MODULE := facedetect
LOCAL_LDLIBS += -llog -ldl -lm -lmediandk
LOCAL_LDLIBS += -lEGL -lGLESv2 -landroid
LOCAL_LDLIBS += -ljnigraphics -fopenmp
LOCAL_CFLAGS += -fpermissive
LOCAL_CPPFLAGS += -fpermissive
#LOCAL_CFLAGS += -ftree-vectorizer-verbose=2
LOCAL_CPPFLAGS += -std=c++17
LOCAL_LDLIBS += -lstdc++
LOCAL_C_INCLUDES += $(LOCAL_PATH)
LOCAL_C_INCLUDES += include
LOCAL_C_INCLUDES += base
LOCAL_C_INCLUDES += aisdk
LOCAL_C_INCLUDES += digit
LOCAL_C_INCLUDES += render
LOCAL_C_INCLUDES += android
LOCAL_C_INCLUDES += third/arm/include
LOCAL_C_INCLUDES += third/arm/include/ncnn
LOCAL_C_INCLUDES += third/opencv-mobile-4.6.0-android/sdk/native/jni/include/
LOCAL_C_INCLUDES += third/ncnn-20221128-android-vulkan-shared/arm64-v8a/include/ncnn
include $(BUILD_SHARED_LIBRARY)

View File

@@ -0,0 +1,169 @@
project(scrfdncnn)
cmake_minimum_required(VERSION 3.10.2)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -funwind-tables")
set(OpenCV_DIR ${CMAKE_SOURCE_DIR}/third/opencv-mobile-4.6.0-android/sdk/native/jni)
find_package(OpenCV REQUIRED core imgproc highgui)
#set(ncnn_DIR ${CMAKE_SOURCE_DIR}/third/ncnn-20221128-android-vulkan-shared/${ANDROID_ABI}/lib/cmake/ncnn)
set(ncnn_DIR ${CMAKE_SOURCE_DIR}/third/ncnn-20231027-android-shared/${ANDROID_ABI}/lib/cmake/ncnn)
find_package(ncnn REQUIRED)
option(USE_OPENCV "shared library support" TRUE)
option(USE_NCNN "shared library support" TRUE)
include_directories(
include
base
render
aisdk
aes
digit
android
third/arm/include
third/arm/include/turbojpeg
)
add_library(scrfdncnn SHARED
android/DigitJni.cpp
android/MsgcbJni.cpp
android/JniHelper.cpp
aisdk/jmat.cpp
aisdk/wavreader.cpp
aisdk/wenet.cpp
aisdk/aimodel.cpp
aisdk/scrfd.cpp
aisdk/pfpld.cpp
aisdk/munet.cpp
aisdk/malpha.cpp
aisdk/wavcache.cpp
aisdk/blendgram.cpp
aisdk/face_utils.cpp
aisdk/netwav.cpp
digit/looper.cpp
digit/netcurl.cpp
digit/GRender.cpp
digit/GDigit.cpp
digit/dispatchqueue.cpp
render/EglRenderer.cpp
render/RgbVideoRenderer.cpp
render/SurfaceVideoRenderer.cpp
render/RenderHelper.cpp
base/BaseRenderHelper.cpp
base/AudioTrack.cpp
render/AudioRenderer.cpp
render/GlesProgram.cpp
base/Log.cpp
base/FrameSource.cpp
base/MediaData.cpp
base/MessageSource.cpp
base/MessageHelper.cpp
base/LoopThread.cpp
base/XThread.cpp
base/XTick.c
base/cJSON.c
base/dh_mem.c
digit/grtcfg.c
base/LoopThreadHelper.cpp
base/coffeecatch.c
aes/aes_cbc.c aes/aes_core.c aes/aes_ecb.c aes/base64.c aes/cbc128.c aes/gj_aes.c
aes/aesmain.c
)
add_library(turbojpeg STATIC IMPORTED)
set_target_properties(turbojpeg
PROPERTIES IMPORTED_LOCATION
${CMAKE_SOURCE_DIR}/third/arm/${ANDROID_ABI}/libturbojpeg.a)
add_library(libjpeg STATIC IMPORTED)
set_target_properties(libjpeg
PROPERTIES IMPORTED_LOCATION
${CMAKE_SOURCE_DIR}/third/arm/${ANDROID_ABI}/libjpeg.a)
#add_library(pplcommon STATIC IMPORTED)
#set_target_properties(pplcommon
# PROPERTIES IMPORTED_LOCATION
# ${CMAKE_SOURCE_DIR}/third/arm/${ANDROID_ABI}/libpplcommon_static.a)
#add_library(pplcv STATIC IMPORTED)
#set_target_properties(pplcv
# PROPERTIES IMPORTED_LOCATION
# ${CMAKE_SOURCE_DIR}/third/arm/${ANDROID_ABI}/libpplcv_static.a)
add_library(curl STATIC IMPORTED)
set_target_properties(curl
PROPERTIES IMPORTED_LOCATION
${CMAKE_SOURCE_DIR}/third/arm/${ANDROID_ABI}/libcurl.a)
add_library(ssl STATIC IMPORTED)
set_target_properties(ssl
PROPERTIES IMPORTED_LOCATION
${CMAKE_SOURCE_DIR}/third/arm/${ANDROID_ABI}/libssl.a)
add_library(crypto STATIC IMPORTED)
set_target_properties(crypto
PROPERTIES IMPORTED_LOCATION
${CMAKE_SOURCE_DIR}/third/arm/${ANDROID_ABI}/libcrypto.a)
add_library(avcodec STATIC IMPORTED)
set_target_properties(avcodec
PROPERTIES IMPORTED_LOCATION
${CMAKE_SOURCE_DIR}/third/arm/${ANDROID_ABI}/ffmpeg-lite/libavcodec.a)
add_library(avformat STATIC IMPORTED)
set_target_properties(avformat
PROPERTIES IMPORTED_LOCATION
${CMAKE_SOURCE_DIR}/third/arm/${ANDROID_ABI}/ffmpeg-lite/libavformat.a)
add_library(avutil STATIC IMPORTED)
set_target_properties(avutil
PROPERTIES IMPORTED_LOCATION
${CMAKE_SOURCE_DIR}/third/arm/${ANDROID_ABI}/ffmpeg-lite/libavutil.a)
add_library(swresample STATIC IMPORTED)
set_target_properties(swresample
PROPERTIES IMPORTED_LOCATION
${CMAKE_SOURCE_DIR}/third/arm/${ANDROID_ABI}/ffmpeg-lite/libswresample.a)
add_library(swscale STATIC IMPORTED)
set_target_properties(swscale
PROPERTIES IMPORTED_LOCATION
${CMAKE_SOURCE_DIR}/third/arm/${ANDROID_ABI}/ffmpeg-lite/libswscale.a)
find_library(log-lib log)
add_library(onnx-lib SHARED IMPORTED)
set_target_properties(
onnx-lib
PROPERTIES IMPORTED_LOCATION
${CMAKE_SOURCE_DIR}/third/arm/${ANDROID_ABI}/libonnxruntime.so)
target_link_libraries(scrfdncnn
ncnn
${OpenCV_LIBS}
${log-lib}
onnx-lib
camera2ndk
mediandk
libjpeg
turbojpeg
avformat
avcodec
avutil
swresample
swscale
curl
ssl
crypto
-landroid
-lmediandk
-lEGL
-lGLESv2
-lm -lz
)

View File

@@ -0,0 +1,41 @@
#ifndef HEADER_AES_H
# define HEADER_AES_H
# include <stddef.h>
# define AES_ENCRYPT 1
# define AES_DECRYPT 0
# define AES_MAXNR 14
# define AES_BLOCK_SIZE 16
struct aes_key_st {
# ifdef AES_LONG
unsigned long rd_key[4 * (AES_MAXNR + 1)];
# else
unsigned int rd_key[4 * (AES_MAXNR + 1)];
# endif
int rounds;
};
typedef struct aes_key_st AES_KEY;
int AES_set_encrypt_key(const unsigned char *userKey, const int bits, AES_KEY *key);
int AES_set_decrypt_key(const unsigned char *userKey, const int bits, AES_KEY *key);
void AES_encrypt(const unsigned char *in, unsigned char *out, const AES_KEY *key);
void AES_decrypt(const unsigned char *in, unsigned char *out, const AES_KEY *key);
void AES_ecb_encrypt(const unsigned char *in, unsigned char *out, const AES_KEY *key,
const int enc);
void AES_cbc_encrypt(const unsigned char *in, unsigned char *out,
size_t length, const AES_KEY *key,
unsigned char *ivec, const int enc);
#endif

View File

@@ -0,0 +1,23 @@
/*
* Copyright 2002-2016 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the OpenSSL license (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#include "aes.h"
#include "modes.h"
void AES_cbc_encrypt(const unsigned char *in, unsigned char *out,
size_t len, const AES_KEY *key,
unsigned char *ivec, const int enc)
{
if (enc)
CRYPTO_cbc128_encrypt(in, out, len, key, ivec,
(block128_f) AES_encrypt);
else
CRYPTO_cbc128_decrypt(in, out, len, key, ivec, (block128_f) AES_decrypt);
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,24 @@
/*
* Copyright 2002-2016 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the OpenSSL license (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#include <assert.h>
#include "aes.h"
#include "aes_locl.h"
void AES_ecb_encrypt(const unsigned char *in, unsigned char *out, const AES_KEY *key, const int enc)
{
assert(in && out && key);
assert((AES_ENCRYPT == enc) || (AES_DECRYPT == enc));
if (AES_ENCRYPT == enc)
AES_encrypt(in, out, key);
else
AES_decrypt(in, out, key);
}

View File

@@ -0,0 +1,42 @@
/*
* Copyright 2002-2016 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the OpenSSL license (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#ifndef HEADER_AES_LOCL_H
# define HEADER_AES_LOCL_H
//# include <e_os2.h>
# include <stdio.h>
# include <stdlib.h>
# include <string.h>
# if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64))
# define SWAP(x) (_lrotl(x, 8) & 0x00ff00ff | _lrotr(x, 8) & 0xff00ff00)
# define GETU32(p) SWAP(*((u32 *)(p)))
# define PUTU32(ct, st) { *((u32 *)(ct)) = SWAP((st)); }
# else
# define GETU32(pt) (((u32)(pt)[0] << 24) ^ ((u32)(pt)[1] << 16) ^ ((u32)(pt)[2] << 8) ^ ((u32)(pt)[3]))
# define PUTU32(ct, st) { (ct)[0] = (u8)((st) >> 24); (ct)[1] = (u8)((st) >> 16); (ct)[2] = (u8)((st) >> 8); (ct)[3] = (u8)(st); }
# endif
# ifdef AES_LONG
typedef unsigned long u32;
# else
typedef unsigned int u32;
# endif
typedef unsigned short u16;
typedef unsigned char u8;
# define MAXKC (256/32)
# define MAXKB (256/8)
# define MAXNR 14
/* This controls loop-unrolling in aes_core.c */
# undef FULL_UNROLL
#endif /* !HEADER_AES_LOCL_H */

View File

@@ -0,0 +1,111 @@
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include "gj_aes.h"
#include "aesmain.h"
int mainenc(int enc,char* infn,char* outfn){
char result[255] ;
memset(result,0,255);
char* key = "yymrjzbwyrbjszrk";
char* aiv = "yymrjzbwyrbjszrk";
int base64 = 1;
int outlen = 0;
int encrst = 0;
char* fn1 = infn;
char* fn2 = outfn;
FILE* fr = fopen(fn1,"rb");
FILE* fw = fopen(fn2,"wb");
while(1){
if(!fr){
encrst = -1001;
break;
}
if(!fw){
encrst = -1002;
break;
}
gj_aesc_t* aesc = NULL;
init_aesc(key,aiv,enc,&aesc);
uint64_t size = 0;
uint64_t realsize = 0;
if(enc){
fwrite("gjdigits",1,8,fw);
fwrite(&size,1,8,fw);
fwrite(&size,1,8,fw);
fwrite(&size,1,8,fw);
while(!feof(fr)){
char data[16];
memset(data,0,16);
uint64_t rst = fread(data,1,16,fr);
if(rst){
size +=rst;
do_aesc(aesc,data,16,result,&outlen);
fwrite(result,1,outlen,fw);
}
}
fseek(fw,8,0);
fwrite(&size,1,8,fw);
}else{
uint64_t rst = fread(result,1,32,fr);
if(!rst){
encrst = -1003;
break;
}
if((result[0]!='g')||(result[1]!='j')){
encrst = -1004;
break;
}
uint64_t *psize = (uint64_t*)(result+8);
realsize = *psize;
if(realsize>1034*1024*1024){
encrst = -1005;
break;
}
while(!feof(fr)){
char data[16];
memset(data,0,16);
uint64_t rst = fread(data,1,16,fr);
if(rst){
size +=rst;
do_aesc(aesc,data,16,result,&outlen);
if(size>realsize){
outlen -= (size-realsize);
//printf("===%lu > %lu rst %lu %d outlen \n",size,realsize,rst,outlen);
}
fwrite(result,1,outlen,fw);
}
}
}
break;
}
if(fr) fclose(fr);
if(fw) fclose(fw);
return encrst;
}
#ifdef TEST
int main(int argc,char** argv){
if(argc<4){
printf("gaes enc|dec filein fileout\n");
return 0;
}
char k = argv[1][0];
if(k=='e'){
int rst = mainenc(1,argv[2],argv[3]);
printf("====enc %s to %s rst %d\n",argv[2],argv[3],rst);
return rst;
}else if(k=='d'){
int rst = mainenc(0,argv[2],argv[3]);
printf("====dec %s to %s rst %d\n",argv[2],argv[3],rst);
return rst;
}else{
printf("gaes enc|dec filein fileout\n");
return 0;
}
}
#endif

View File

@@ -0,0 +1,16 @@
#ifndef __AESMAIN_H
#define __AESMAIN_H
#include "gj_dll.h"
#ifdef __cplusplus
extern "C" {
#endif
int mainenc(int enc,char* infn,char* outfn);
#ifdef __cplusplus
}
#endif
#endif

View File

@@ -0,0 +1,164 @@
/* This is a public domain base64 implementation written by WEI Zhicheng. */
#include "base64.h"
#define BASE64_PAD '='
#define BASE64DE_FIRST '+'
#define BASE64DE_LAST 'z'
/* BASE 64 encode table */
static const char base64en[] = {
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
'w', 'x', 'y', 'z', '0', '1', '2', '3',
'4', '5', '6', '7', '8', '9', '+', '/',
};
/* ASCII order for BASE 64 decode, 255 in unused character */
static const unsigned char base64de[] = {
/* nul, soh, stx, etx, eot, enq, ack, bel, */
255, 255, 255, 255, 255, 255, 255, 255,
/* bs, ht, nl, vt, np, cr, so, si, */
255, 255, 255, 255, 255, 255, 255, 255,
/* dle, dc1, dc2, dc3, dc4, nak, syn, etb, */
255, 255, 255, 255, 255, 255, 255, 255,
/* can, em, sub, esc, fs, gs, rs, us, */
255, 255, 255, 255, 255, 255, 255, 255,
/* sp, '!', '"', '#', '$', '%', '&', ''', */
255, 255, 255, 255, 255, 255, 255, 255,
/* '(', ')', '*', '+', ',', '-', '.', '/', */
255, 255, 255, 62, 255, 255, 255, 63,
/* '0', '1', '2', '3', '4', '5', '6', '7', */
52, 53, 54, 55, 56, 57, 58, 59,
/* '8', '9', ':', ';', '<', '=', '>', '?', */
60, 61, 255, 255, 255, 255, 255, 255,
/* '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G', */
255, 0, 1, 2, 3, 4, 5, 6,
/* 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', */
7, 8, 9, 10, 11, 12, 13, 14,
/* 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', */
15, 16, 17, 18, 19, 20, 21, 22,
/* 'X', 'Y', 'Z', '[', '\', ']', '^', '_', */
23, 24, 25, 255, 255, 255, 255, 255,
/* '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', */
255, 26, 27, 28, 29, 30, 31, 32,
/* 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', */
33, 34, 35, 36, 37, 38, 39, 40,
/* 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', */
41, 42, 43, 44, 45, 46, 47, 48,
/* 'x', 'y', 'z', '{', '|', '}', '~', del, */
49, 50, 51, 255, 255, 255, 255, 255
};
unsigned int
gjbase64_encode(const unsigned char *in, unsigned int inlen, char *out)
{
int s;
unsigned int i;
unsigned int j;
unsigned char c;
unsigned char l;
s = 0;
l = 0;
for (i = j = 0; i < inlen; i++) {
c = in[i];
switch (s) {
case 0:
s = 1;
out[j++] = base64en[(c >> 2) & 0x3F];
break;
case 1:
s = 2;
out[j++] = base64en[((l & 0x3) << 4) | ((c >> 4) & 0xF)];
break;
case 2:
s = 0;
out[j++] = base64en[((l & 0xF) << 2) | ((c >> 6) & 0x3)];
out[j++] = base64en[c & 0x3F];
break;
}
l = c;
}
switch (s) {
case 1:
out[j++] = base64en[(l & 0x3) << 4];
out[j++] = BASE64_PAD;
out[j++] = BASE64_PAD;
break;
case 2:
out[j++] = base64en[(l & 0xF) << 2];
out[j++] = BASE64_PAD;
break;
}
out[j] = 0;
return j;
}
unsigned int
gjbase64_decode(const char *in, unsigned int inlen, unsigned char *out)
{
unsigned int i;
unsigned int j;
unsigned char c;
if (inlen & 0x3) {
return 0;
}
for (i = j = 0; i < inlen; i++) {
if (in[i] == BASE64_PAD) {
break;
}
if (in[i] < BASE64DE_FIRST || in[i] > BASE64DE_LAST) {
return 0;
}
c = base64de[(unsigned char)in[i]];
if (c == 255) {
return 0;
}
switch (i & 0x3) {
case 0:
out[j] = (c << 2) & 0xFF;
break;
case 1:
out[j++] |= (c >> 4) & 0x3;
out[j] = (c & 0xF) << 4;
break;
case 2:
out[j++] |= (c >> 2) & 0xF;
out[j] = (c & 0x3) << 6;
break;
case 3:
out[j++] |= c;
break;
}
}
return j;
}

View File

@@ -0,0 +1,29 @@
#ifndef BASE64_H
#define BASE64_H
#define BASE64_ENCODE_OUT_SIZE(s) ((unsigned int)((((s) + 2) / 3) * 4 + 1))
#define BASE64_DECODE_OUT_SIZE(s) ((unsigned int)(((s) / 4) * 3))
#ifdef __cplusplus
extern "C"{
#endif
/*
* out is null-terminated encode string.
* return values is out length, exclusive terminating `\0'
*/
unsigned int
gjbase64_encode(const unsigned char *in, unsigned int inlen, char *out);
/*
* return values is out length
*/
unsigned int
gjbase64_decode(const char *in, unsigned int inlen, unsigned char *out);
#ifdef __cplusplus
}
#endif
#endif /* BASE64_H */

View File

@@ -0,0 +1,161 @@
/*
* Copyright 2008-2016 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the OpenSSL license (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
// #include <openssl/crypto.h>
#include "modes.h"
#include <string.h>
#if !defined(STRICT_ALIGNMENT) && !defined(PEDANTIC)
# define STRICT_ALIGNMENT 0
#endif
void CRYPTO_cbc128_encrypt(const unsigned char *in, unsigned char *out,
size_t len, const void *key,
unsigned char ivec[16], block128_f block)
{
size_t n;
const unsigned char *iv = ivec;
if (len == 0)
return;
#if !defined(OPENSSL_SMALL_FOOTPRINT)
if (STRICT_ALIGNMENT &&
((size_t)in | (size_t)out | (size_t)ivec) % sizeof(size_t) != 0) {
while (len >= 16) {
for (n = 0; n < 16; ++n)
out[n] = in[n] ^ iv[n];
(*block) (out, out, key);
iv = out;
len -= 16;
in += 16;
out += 16;
}
} else {
while (len >= 16) {
for (n = 0; n < 16; n += sizeof(size_t))
*(size_t *)(out + n) =
*(size_t *)(in + n) ^ *(size_t *)(iv + n);
(*block) (out, out, key);
iv = out;
len -= 16;
in += 16;
out += 16;
}
}
#endif
while (len) {
for (n = 0; n < 16 && n < len; ++n)
out[n] = in[n] ^ iv[n];
for (; n < 16; ++n)
out[n] = iv[n];
(*block) (out, out, key);
iv = out;
if (len <= 16)
break;
len -= 16;
in += 16;
out += 16;
}
memcpy(ivec, iv, 16);
}
void CRYPTO_cbc128_decrypt(const unsigned char *in, unsigned char *out,
size_t len, const void *key,
unsigned char ivec[16], block128_f block)
{
size_t n;
union {
size_t t[16 / sizeof(size_t)];
unsigned char c[16];
} tmp;
if (len == 0)
return;
#if !defined(OPENSSL_SMALL_FOOTPRINT)
if (in != out) {
const unsigned char *iv = ivec;
if (STRICT_ALIGNMENT &&
((size_t)in | (size_t)out | (size_t)ivec) % sizeof(size_t) != 0) {
while (len >= 16) {
(*block) (in, out, key);
for (n = 0; n < 16; ++n)
out[n] ^= iv[n];
iv = in;
len -= 16;
in += 16;
out += 16;
}
} else if (16 % sizeof(size_t) == 0) { /* always true */
while (len >= 16) {
size_t *out_t = (size_t *)out, *iv_t = (size_t *)iv;
(*block) (in, out, key);
for (n = 0; n < 16 / sizeof(size_t); n++)
out_t[n] ^= iv_t[n];
iv = in;
len -= 16;
in += 16;
out += 16;
}
}
memcpy(ivec, iv, 16);
} else {
if (STRICT_ALIGNMENT &&
((size_t)in | (size_t)out | (size_t)ivec) % sizeof(size_t) != 0) {
unsigned char c;
while (len >= 16) {
(*block) (in, tmp.c, key);
for (n = 0; n < 16; ++n) {
c = in[n];
out[n] = tmp.c[n] ^ ivec[n];
ivec[n] = c;
}
len -= 16;
in += 16;
out += 16;
}
} else if (16 % sizeof(size_t) == 0) { /* always true */
while (len >= 16) {
size_t c, *out_t = (size_t *)out, *ivec_t = (size_t *)ivec;
const size_t *in_t = (const size_t *)in;
(*block) (in, tmp.c, key);
for (n = 0; n < 16 / sizeof(size_t); n++) {
c = in_t[n];
out_t[n] = tmp.t[n] ^ ivec_t[n];
ivec_t[n] = c;
}
len -= 16;
in += 16;
out += 16;
}
}
}
#endif
while (len) {
unsigned char c;
(*block) (in, tmp.c, key);
for (n = 0; n < 16 && n < len; ++n) {
c = in[n];
out[n] = tmp.c[n] ^ ivec[n];
ivec[n] = c;
}
if (len <= 16) {
for (; n < 16; ++n)
ivec[n] = in[n];
break;
}
len -= 16;
in += 16;
out += 16;
}
}

View File

@@ -0,0 +1,213 @@
#include "gaes_stream.h"
#include <cstring>
#include <iostream>
#include <fstream>
#include <vector>
#include <cstdio>
#include <cstdlib>
#include "gj_aes.h"
class GaesIStreamBuf final: public std::streambuf
{
private:
char *m_inbuf;
size_t m_inbufsize;
bool m_owns_inbuf;
char *m_leftbuf;
FILE *file;
uint64_t cur_size;
uint64_t file_size;
gj_aesc_t* aesc ;
protected:
virtual std::streambuf* setbuf(char *s, std::streamsize n){
setg(0, 0, 0);
if (m_owns_inbuf) {
delete [] m_inbuf;
}
m_inbufsize = n;
if (s) {
m_inbuf = s;
m_owns_inbuf = false;
} else {
m_inbuf = new char[m_inbufsize];
m_leftbuf = new char[m_inbufsize];
m_owns_inbuf = true;
}
return this;
}
virtual int sync(){
int result = 0;
return result;
}
virtual int underflow() override{
int __c = traits_type::eof();
if (!file) return __c;
if(cur_size>=file_size){
printf("===eof %ld ===%ld\n",cur_size,file_size);
return __c;
}
bool initial = false;
if (eback() == 0) {
setg(m_inbuf, m_inbuf + m_inbufsize, m_inbuf + m_inbufsize);
initial = true;
}
const size_t unget_sz = initial ? 0 : std::min<size_t>((egptr() - eback()) / 2, 4);
if (gptr() == egptr()) {
memmove(eback(), egptr() - unget_sz, unget_sz);
size_t nmemb = static_cast<size_t>(egptr() - eback() - unget_sz);
char* pdst = eback() + unget_sz;
int modb = nmemb % 16;
size_t leftb = nmemb - modb;
char* pbuf = m_leftbuf;
size_t leftf = file_size - cur_size;
if(leftb>leftf)leftb=leftf;
memset(pbuf,0,m_inbufsize);
size_t rd = fread(pbuf, 1, leftb, file);
//printf("%d-%ld-%ld----------------%ld--%ld#\n",cur_size,file_size,modb,nmemb,rd);
//ssize_t readed = read(m_fd, eback() + unget_sz, nmemb);
if(rd>0){
cur_size += rd;
int cnt = leftb /16;
int k;
for(k=0;k<cnt;k++){
int outlen = 0;
do_aesc(aesc,pbuf,16,pdst,&outlen);
pbuf += 16;
pdst += 16;
}
setg(eback(), eback() + unget_sz, eback() + unget_sz + rd);
__c = traits_type::to_int_type(*gptr());
}
} else {
__c = traits_type::to_int_type(*gptr());
}
return __c;
}
public:
GaesIStreamBuf(std::string& filename) :m_inbuf(0), m_inbufsize(0), m_owns_inbuf(false){
setbuf(0, 1024);
cur_size = 0;
file = fopen(filename.c_str(), "rb");
fseek(file, 0, SEEK_END);
file_size = ftell(file); //获取音频文件大小
fseek(file, 0, SEEK_SET);
char* key = "yymrjzbwyrbjszrk";
char* aiv = "yymrjzbwyrbjszrk";
init_aesc(key,aiv,0,&this->aesc);
char head[50];
memset(head,0,50);
uint64_t rst = fread(head,1,8,file);
rst = fread(&cur_size,1,8,file);
printf("===head %s size %ld\n",head,cur_size);
rst = fread(head,1,16,file);
cur_size = 32;
}
~GaesIStreamBuf(){
close();
if (m_owns_inbuf) {
delete[] m_inbuf;
}
}
void close(){
if(aesc){
free_aesc(&this->aesc);
}
if (file){
fclose(file);
file = NULL;
}
}
};
GaesIStream::GaesIStream(std::string filename):
std::istream(new GaesIStreamBuf(filename)){
}
GaesIStream::~GaesIStream()
{
delete rdbuf();
}
#ifdef TEST
int maindec(int argc,char** argv){
std::string filename(argv[1]);// = "test.enc";
//std::string filename = "final.mdlenc";
GaesIStream fin(filename);
//std::string fn2 = "final.mdldec";
std::string fn2(argv[2]);// = "test.dec";
std::ofstream fout(fn2,std::ios::binary);
char buf[1024];
int rd = 0;
while(!fin.eof()){
//while((rd = fin.read(buf,16))>0){
//printf("===rd %ld\n",rd);
fin.read(buf,16);
fout.write(buf,16);
}
//char ch;
//while (fin.get(ch)) {
//printf("+");
//fout << ch;
//}
return 0;
}
int mainenc(int argc,char** argv){
char result[255] ;
memset(result,0,255);
char* key = "yymrjzbwyrbjszrk";
char* aiv = "yymrjzbwyrbjszrk";
int base64 = 1;
int outlen = 0;
gj_aesc_t* aesc = NULL;
init_aesc(key,aiv,1,&aesc);
char* fn1 = argv[1];
char* fn2 = argv[2];
FILE* fr = fopen(fn1,"rb");
FILE* fw = fopen(fn2,"wb");
fwrite("abcdefgh",1,8,fw);
uint64_t size = 0;
fwrite(&size,1,8,fw);
fwrite(&size,1,8,fw);
fwrite(&size,1,8,fw);
while(!feof(fr)){
char data[16];
memset(data,0,16);
uint64_t rst = fread(data,1,16,fr);
printf("===rst %d\n",rst);
if(rst){
size +=rst;
do_aesc(aesc,data,16,result,&outlen);
printf("===out %d\n",outlen);
fwrite(result,1,16,fw);
}
}
fseek(fw,8,0);
fwrite(&size,1,8,fw);
fclose(fr);
fclose(fw);
return 0;
}
int main(int argc,char** argv){
if(argc<4){
return mainenc(argc,argv);
}else{
return maindec(argc,argv);
}
}
#endif

View File

@@ -0,0 +1,22 @@
#ifndef COMPRESSED_STREAMS_ZSTD_STREAM_H
#define COMPRESSED_STREAMS_ZSTD_STREAM_H
#include <iostream>
class GaesIStream: public std::istream
{
public:
GaesIStream(std::string filename);
virtual ~GaesIStream();
};
#endif // COMPRESSED_STREAMS_ZSTD_STREAM_H

View File

@@ -0,0 +1,69 @@
#include <stdlib.h>
#include <string.h>
#include "gj_aes.h"
#include "base64.h"
#include "aes.h"
struct gj_aesc_s{
char key[16];
char iv[16];
int enc;
AES_KEY *aeskey;
};
int free_aesc(gj_aesc_t** paesc){
if(!paesc||!*paesc)return -1;
if((*paesc)->aeskey)free((*paesc)->aeskey);
free(*paesc);
*paesc = NULL;
return 0;
}
int init_aesc(char* key,char* iv,int enc,gj_aesc_t** paesc){
if(strlen(key)!=16) return -1;
if(strlen(iv)!=16) return -2;
gj_aesc_t* aesc = (gj_aesc_t*)malloc(sizeof(gj_aesc_t));
int k;
for(k=0;k<16;k++){
aesc->key[k]=key[k];
aesc->iv[k]=iv[k];
}
aesc->aeskey = (AES_KEY*)malloc(sizeof(AES_KEY));
aesc->enc = enc;
if(enc){
AES_set_encrypt_key((const unsigned char*)aesc->key, 128, aesc->aeskey);
}else{
AES_set_decrypt_key((const unsigned char*)aesc->key, 128, aesc->aeskey);
}
*paesc = aesc;
return 0;
}
int do_aesc(gj_aesc_t* aesc,char* in,int inlen,char* out,int* outlen){
char* psrc = in;
char* pdest = out;
int cnt = 0;
int left=inlen;
while(left>0){
AES_cbc_encrypt((const unsigned char*)psrc,(unsigned char*)pdest,16,aesc->aeskey,(unsigned char*)aesc->iv,aesc->enc);
psrc += 16;
pdest += 16;
left -= 16;
cnt += 16;
}
*outlen = cnt;
return 0;
}
int do_base64(int enc,char* in,int inlen,char* out,int* outlen){
if(enc){
gjbase64_encode((unsigned char*)in,inlen,out);
*outlen = strlen(out);
}else{
*outlen = gjbase64_decode(in,inlen,(unsigned char*)out);
}
return 0;
}

View File

@@ -0,0 +1,22 @@
#ifndef __GJ_AES_H__
#define __GJ_AES_H__
#include "gj_dll.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct gj_aesc_s gj_aesc_t;
GJLIBAPI int free_aesc(gj_aesc_t** paesc);
GJLIBAPI int init_aesc(char* key,char* iv,int enc,gj_aesc_t** paesc);
GJLIBAPI int do_aesc(gj_aesc_t* aesc,char* in,int inlen,char* out,int* outlen);
GJLIBAPI int do_base64(int enc,char* in,int inlen,char* out,int* outlen);
#ifdef __cplusplus
}
#endif
#endif

View File

@@ -0,0 +1,21 @@
#ifndef __GJ_DLL_H__
#define __GJ_DLL_H__
#ifdef __cplusplus
extern "C" {
#endif
#define GJLIB_EXPORT 1
#if defined(GJLIB_EXPORT)
#if defined _WIN32 || defined __CYGWIN__
#define GJLIBAPI __declspec(dllexport)
#else
#define GJLIBAPI __attribute__((visibility("default")))
#endif
#else
#define GJLIBAPI
#endif
#ifdef __cplusplus
}
#endif
#endif

View File

@@ -0,0 +1,3 @@
all:
g++ -fPIC -o gjaesmain -g aesmain.c \
aes_cbc.c aes_core.c aes_ecb.c cbc128.c base64.c gj_aes.c -lm --std=c++11 -I. -DTEST

View File

@@ -0,0 +1,22 @@
#ifndef HEADER_MODES_H
# define HEADER_MODES_H
# include <stddef.h>
typedef void (*block128_f) (const unsigned char in[16],
unsigned char out[16], const void *key);
typedef void (*cbc128_f) (const unsigned char *in, unsigned char *out,
size_t len, const void *key,
unsigned char ivec[16], int enc);
void CRYPTO_cbc128_encrypt(const unsigned char *in, unsigned char *out,
size_t len, const void *key,
unsigned char ivec[16], block128_f block);
void CRYPTO_cbc128_decrypt(const unsigned char *in, unsigned char *out,
size_t len, const void *key,
unsigned char ivec[16], block128_f block);
#endif

View File

@@ -0,0 +1,24 @@
#pragma once
//#define MFCC_OFFSET 6436
#define MFCC_OFFSET 6400
//##define MFCC_OFFSET 0
#define MFCC_DEFRMS 0.1f
#define MFCC_FPS 25
#define MFCC_RATE 16000
//#define MFCC_WAVCHUNK 960000
#define MFCC_WAVCHUNK 560000
//#define MFCC_WAVCHUNK 512
//#define MFCC_MELBASE 6001
#define MFCC_MELBASE 3501
#define MFCC_MELCHUNK 80
//#define MFCC_MELCHUNK 20
//#define MFCC_BNFBASE 1499
#define MFCC_BNFBASE 874
#define MFCC_BNFCHUNK 256
//input==== NodeArg(name='speech', type='tensor(float)', shape=['B', 'T', 80])
//input==== NodeArg(name='speech_lengths', type='tensor(int32)', shape=['B'])
//output==== NodeArg(name='encoder_out', type='tensor(float)', shape=['B', 'T_OUT', 'Addencoder_out_dim_2'])

View File

@@ -0,0 +1,438 @@
#include "aimodel.h"
#include <stdlib.h>
#include <string.h>
void AiCfg::dump(){
int incnt = size_inputs.size();
int outcnt = size_outputs.size();
std::cout<<"======in onnx:"<<incnt<<std::endl;
//std::cout<<"======in onnx:"<<name_inputs.size()<<std::endl;
for(int k=0;k<incnt;k++){
//std::string sname(name_inputs[k]);
//std::cout<<"in name:"<<sname<<std::endl;
std::cout<<"size input:"<<size_inputs[k]<<std::endl;
for(int m=0;m<shape_inputs[k].size();m++)
std::cout<<"shape :"<<shape_inputs[k][m]<<std::endl;
std::cout<<"kind input:"<<kind_inputs[k]<<std::endl;
}
std::cout<<"=========out onnx:"<<outcnt<<std::endl;
//std::cout<<"======in onnx:"<<name_outputs.size()<<std::endl;
for(int k=0;k<outcnt;k++){
//std::string sname(name_outputs[k]);
//std::cout<<"out name:"<<sname<<std::endl;
std::cout<<"size output:"<<size_outputs[k]<<std::endl;
for(int m=0;m<shape_outputs[k].size();m++)
std::cout<<"shape :"<<shape_outputs[k][m]<<std::endl;
std::cout<<"kind outintpu:"<<kind_outputs[k]<<std::endl;
}
}
int AiCfg::inShape(int inx,int dim,int val){
if(inx>=shape_inputs.size())return -1;
if(dim>=shape_inputs[inx].size())return -2;
shape_inputs[inx][dim] = val;
int size = 1;
for(int k=0;k<shape_inputs[inx].size();k++){
size *= shape_inputs[inx][dim];
}
if(size>0){
size_inputs[inx] = size;
}
return 0;
}
int AiCfg::outShape(int inx,int dim,int val){
if(inx>=shape_outputs.size())return -1;
if(dim>=shape_outputs[inx].size())return -2;
shape_outputs[inx][dim] = val;
int size = 1;
for(int k=0;k<shape_outputs[inx].size();k++){
size *= shape_outputs[inx][dim];
}
if(size>0){
size_outputs[inx] = size;
}
return 0;
}
AiCfg AiCfg::clone(){
AiCfg onecfg;
AiCfg* cfg = &onecfg;//new AiCfg();
//
cfg->names.assign(names.begin(),names.end());
cfg->kind_inputs.assign(kind_inputs.begin(),kind_inputs.end());
cfg->name_inputs.assign(name_inputs.begin(),name_inputs.end());
cfg->size_inputs.assign(size_inputs.begin(),size_inputs.end());
//cfg->m_sizeinput = m_sizeinput;
for(int k=0;k<shape_inputs.size();k++){
std::vector<int64_t> shape(shape_inputs[k]);
cfg->shape_inputs.push_back(shape);
}
cfg->kind_outputs.assign(kind_outputs.begin(),kind_outputs.end());
cfg->name_outputs.assign(name_outputs.begin(),name_outputs.end());
cfg->size_outputs.assign(size_outputs.begin(),size_outputs.end());
//cfg->m_sizeoutput = m_sizeoutput;
for(int k=0;k<shape_outputs.size();k++){
std::vector<int64_t> shape(shape_outputs[k]);
cfg->shape_outputs.push_back(shape);
}
return onecfg;//cfg;
}
AiModel::AiModel(){
m_cfg = new AiCfg();
}
AiModel::~AiModel(){
delete m_cfg;
}
int AiModel::doInitModel(){
return -1;
}
AiCfg AiModel::config(){
return m_cfg->clone();
}
void AiModel::dump(){
m_cfg->dump();
}
int AiModel::pushName(const char* name,int input){
std::string sname(name);
m_cfg->names.push_back(sname);
if(input){
m_cfg->name_inputs.push_back(m_cfg->names[m_cfg->names.size()-1].c_str());
}else{
m_cfg->name_outputs.push_back(m_cfg->names[m_cfg->names.size()-1].c_str());
}
return 0;
}
int AiModel::initModel(std::string& modelpath){
m_modelPath = modelpath;
m_inited = doInitModel();
printf("===init %d\n",m_inited);
return m_inited;
}
int AiModel::initModel(std::string& binfn,std::string& paramfn){
m_modelbin = binfn;
m_modelparam = paramfn;
m_inited = doInitModel();
//printf("===init %d\n",m_inited);
return m_inited;
}
int AiModel::doRunModel(void** arrin,void** arrout,void* stream,AiCfg* pcfg){
return 0;
}
int AiModel::runModel(void** arrin,void** arrout,void* stream,AiCfg* pcfg){
if(m_inited){
//std::cout<<"model not inited:"<<m_inited<<std::endl;
return -999;
}
return doRunModel(arrin,arrout,stream,pcfg);
}
OnnxModel::OnnxModel(){
}
OnnxModel::OnnxModel(int b,int w,int h):AiModel(){
m_batch = b;
m_width = w;
m_height = h;
}
OnnxModel::~OnnxModel(){
}
int OnnxModel::doInitModel(){
env = Ort::Env(OrtLoggingLevel::ORT_LOGGING_LEVEL_WARNING, "ONNX");
sessionOptions = Ort::SessionOptions();
//sessionOptions.AddConfigEntry("session.load_model_format","ORT");
//std::vector<std::string> availableProviders = Ort::GetAvailableProviders();
//auto cudaAvailable = std::find(availableProviders.begin(), availableProviders.end(), "CUDAExecutionProvider");
//OrtCUDAProviderOptions cudaOption;
//if(cudaAvailable != availableProviders.end()){
//std::cout << "Inference device: GPU" << std::endl;
//sessionOptions.AppendExecutionProvider_CUDA(cudaOption);
//}else{
//std::cout << "Inference device: CPU" << std::endl;
//}
session = Ort::Session(env, m_modelPath.c_str(), sessionOptions);
//Ort::AllocatorWithDefaultOptions allocator;
size_t numInputNodes = session.GetInputCount();
size_t numOutputNodes = session.GetOutputCount();
//std::cout << "input NUM: " << numInputNodes << std::endl;
//std::cout << "Output NUM: " << numOutputNodes << std::endl;
for(int k=0;k<numInputNodes;k++){
//m_cfg->name_inputs.push_back(session.GetInputName(k));//, allocator));
Ort::TypeInfo inputTypeInfo = session.GetInputTypeInfo(k);
auto tensorInfo = inputTypeInfo.GetTensorTypeAndShapeInfo();
auto elemType = tensorInfo.GetElementType();
m_cfg->kind_inputs.push_back((int)elemType);
std::vector<int64_t> inputTensorShape = tensorInfo.GetShape();
if(m_batch&&inputTensorShape.size()){
if(inputTensorShape[0]==-1)inputTensorShape[0]=m_batch;
}
if(m_batch&&(inputTensorShape.size()==4)){
if(inputTensorShape[0]==-1)inputTensorShape[0]=m_batch;
if(inputTensorShape[2]==-1)inputTensorShape[2]=m_height;
if(inputTensorShape[3]==-1)inputTensorShape[3]=m_width;
}
int size = 1;
for (auto shape : inputTensorShape){
size *= shape;
}
m_cfg->shape_inputs.push_back(inputTensorShape);
m_cfg->size_inputs.push_back(size);
}
for(int k=0;k<numOutputNodes;k++){
//m_cfg->name_outputs.push_back(session.GetOutputName(k));//, allocator));
Ort::TypeInfo outputTypeInfo = session.GetOutputTypeInfo(k);
auto tensorInfo = outputTypeInfo.GetTensorTypeAndShapeInfo();
auto elemType = tensorInfo.GetElementType();
m_cfg->kind_outputs.push_back((int)elemType);
std::vector<int64_t> outputTensorShape = tensorInfo.GetShape();
if(m_batch&&outputTensorShape.size()){
if(outputTensorShape[0]==-1)outputTensorShape[0]=m_batch;
}
if(m_batch&&(outputTensorShape.size()==4)){
if(outputTensorShape[0]==-1)outputTensorShape[0]=m_batch;
if(outputTensorShape[2]==-1)outputTensorShape[2]=m_height;
if(outputTensorShape[3]==-1)outputTensorShape[3]=m_width;
}
int size = 1;
for (auto shape : outputTensorShape){
size *= shape;
}
m_cfg->shape_outputs.push_back(outputTensorShape);
m_cfg->size_outputs.push_back(size);
}
return 0;
}
int OnnxModel::doRunModel(void** arrin,void** arrout,void* stream,AiCfg* pcfg){
AiCfg* cfg = pcfg==nullptr?m_cfg:pcfg;
int incnt = cfg->size_inputs.size();
int outcnt = cfg->size_outputs.size();
std::cout<<"run onnx:"<<outcnt<<std::endl;
if(!arrin || !arrout)return -1;
std::vector<Ort::Value> inputTensors;
Ort::MemoryInfo memoryInfo = Ort::MemoryInfo::CreateCpu( OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault);
for(int k=0;k<incnt;k++){
inputTensors.push_back(Ort::Value::CreateTensor( memoryInfo, arrin[k] ,cfg->size_inputs[k]*4 , cfg->shape_inputs[k].data(), cfg->shape_inputs[k].size(), (ONNXTensorElementDataType)cfg->kind_inputs[k] ));
}
std::vector<Ort::Value> outputTensors;
for(int k=0;k<outcnt;k++){
outputTensors.push_back(Ort::Value::CreateTensor( memoryInfo, arrout[k] ,cfg->size_outputs[k]*4 , cfg->shape_outputs[k].data(), cfg->shape_outputs[k].size(),(ONNXTensorElementDataType)cfg->kind_outputs[k] ));
}
this->session.Run(Ort::RunOptions{nullptr}, cfg->names_in, inputTensors.data(), incnt, cfg->names_out, outputTensors.data(),outcnt);
if(1)return 0;
/*
//for(int k=0;k<9;k++)dumpfloat((float*)arrout[k],10);//size_outputs[0]);
std::vector<Ort::Value> aoutputTensors = this->session.Run(Ort::RunOptions{nullptr}, name_inputs.data(), inputTensors.data(), 1, name_outputs.data(), 9);
//bool* pmsk = (bool*)aoutputTensors[1].GetTensorData<bool>();
for(int k=0;k<9;k++){
float* pbnf = (float*)aoutputTensors[0].GetTensorData<float>();
memcpy(arrout[k],pbnf,size_outputs[k]*4);
}
*/
return 0;
}
NcnnModel::NcnnModel():AiModel(){
}
NcnnModel::NcnnModel(int w,int h):AiModel(){
m_width = w;
m_height = h;
}
NcnnModel::~NcnnModel(){
net.clear();
}
int NcnnModel::doInitModel(){
net.clear();
net.load_param(m_modelparam.c_str());
net.load_model(m_modelbin.c_str());
return 0; //
}
int NcnnModel::doRunModel(void** arrin,void** arrout,void* stream,AiCfg* pcfg){
ncnn::Extractor ex = net.create_extractor();
AiCfg* cfg = pcfg==nullptr?m_cfg:pcfg;
int incnt = cfg->size_inputs.size();
int outcnt = cfg->size_outputs.size();
ncnn::Mat inmat[incnt];
for(int k=0;k<incnt;k++){
std::string name = cfg->name_inputs[k];
JMat* mat = (JMat*)arrin[k];
ncnn::Mat in_pack =mat->packingmat();
ex.input(name.c_str(), in_pack);
}
for(int k=0;k<outcnt;k++){
std::string name = cfg->name_outputs[k];
JMat* mat = (JMat*)arrout[k];
ncnn::Mat output;
ex.extract(name.c_str(), output);
ncnn::Mat in_park;
ncnn::convert_packing(output,in_park,3);
int size = mat->width()*mat->height()*3*sizeof(float);
memcpy((uint8_t*)mat->data(),in_park,size);
}
return 0;
}
#ifdef _TENSORRT_
#include "NvInferPlugin.h"
#include "cuda_runtime_api.h"
#include "cuda.h"
#define FPW 4
using namespace nvinfer1;
int TrtModel::doInitModel(){
std::cout<<"deserialize eng !"<<m_modelPath<<std::endl;
bool didInitPlugins = initLibNvInferPlugins(nullptr, "");
std::ifstream cache(m_modelPath,std::ios::binary);
cache.seekg(0,std::ios::end);
int engSize = cache.tellg();
if(!engSize)return -1;
cache.seekg(0,std::ios::beg);
void *modelMem = malloc(engSize);
if(!modelMem)return -2;
cache.read((char*)modelMem,engSize);
cache.close();
std::cout<<"deserialize size!"<<engSize<<std::endl;
IRuntime *runtime = nvinfer1::createInferRuntime(m_logger);
m_engine = runtime->deserializeCudaEngine(modelMem,engSize);
runtime->destroy();
free(modelMem);
if(! m_engine){
std::cout<<"deserialize eng error!"<<std::endl;
return -10;
}
std::cout<<"aaa"<<std::endl;
m_context = m_engine->createExecutionContext();
int bindings = m_engine->getNbBindings();
std::cout<<"bindings!"<<bindings<<std::endl;
for(int k=0;k<bindings;k++){
const char* name = m_engine->getBindingName(k);
std::string sname(name );
bool input = m_engine->bindingIsInput(k);
std::cout<<"name!"<<sname<<"===input"<<input<<std::endl;
Dims dims = m_engine->getBindingDimensions(k);
auto dt = m_engine->getBindingDataType(k);
int tdt = dt==DataType::kFLOAT?ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT:ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32;
std::vector<int64_t> tensorShape;
int dsize = 1;
for(int i = 0 ;i < dims.nbDims; i++){
tensorShape.push_back(dims.d[i]);
dsize *= dims.d[i];
//cout<<"==dsize!"<<dims.d[i]<<endl;
}
//cout<<"==========dsize!"<<dsize<<endl;
int size = dsize*FPW;//size_list[k];
if(input){
m_cfg->kind_inputs.push_back(tdt);
m_cfg->shape_inputs.push_back(tensorShape);
m_cfg->size_inputs.push_back(size);
m_cfg->name_inputs.push_back(name);
}else{
m_cfg->kind_outputs.push_back(tdt);
m_cfg->size_outputs.push_back(size);
m_cfg->name_outputs.push_back(name);
m_cfg->shape_outputs.push_back(tensorShape);
}
}
//getchar();
/*
m_sizeinput = 0;
for(int k=0;k<m_cfg->size_inputs.size();k++){
int size = m_cfg->size_inputs[k];
void* input ;
//int flag = cudaMalloc(&input,size);
//m_inputs.push_back(input);
//m_bindings[k]=input;
m_sizeinput += size;
//printf("===input %d output %d \n",m_sizeinput,m_sizeoutput);
}
std::cout<<"bbb"<<std::endl;
m_sizeoutput = 0;
int offset = m_cfg->size_inputs.size();
for(int k=0;k<m_cfg->size_outputs.size();k++){
int size = m_cfg->size_outputs[k];
void* output ;
//int flag = cudaMalloc(&output,size);
//m_outputs.push_back(output);
//m_bindings[offset+k]=output;
m_sizeoutput += size;
//printf("===input %d output %d \n",m_sizeinput,m_sizeoutput);
}
std::cout<<"ccc"<<std::endl;
*/
return 0;
}
int TrtModel::doRunModel(void** arrin,void** arrout,void* stream,AiCfg* pcfg){
AiCfg* cfg = pcfg==nullptr?m_cfg:pcfg;
int incnt = cfg->size_inputs.size();
int outcnt = cfg->size_outputs.size();
std::cout<<"incnt:"<<incnt<<"==outcnt:"<<outcnt<<std::endl;
std::vector<void*> m_bindings(incnt+outcnt,NULL);
for(int k=0;k<incnt;k++){
std::cout<<"in:"<<cfg->size_inputs[k]<<std::endl;
m_bindings[k] = arrin[k];
}
for(int k=0;k<outcnt;k++){
std::cout<<"out:"<<cfg->size_outputs[k]<<std::endl;
m_bindings[incnt+k] = arrout[k];
}
bool status = m_context->enqueue(1,m_bindings.data(),(cudaStream_t)stream,nullptr);
return status?0:-1;
}
TrtModel::TrtModel(int b,int w,int h):AiModel(){
}
TrtModel::TrtModel():AiModel(){
}
TrtModel::~TrtModel(){
if(m_context){
m_context->destroy();
m_context = nullptr;
}
if(m_engine){
m_engine->destroy();
m_engine = nullptr;
}
}
#endif
/*
int main(int argc,char** argv){
OnnxModel *model=new OnnxModel();
std::string fn = "onnx/mfcc.onnx";
model->initModel(fn);
delete model;
return 0;
}
*/

View File

@@ -0,0 +1,136 @@
#pragma once
#include <vector>
#include <string>
#include <iostream>
#include <fstream>
#include <algorithm>
#include "jmat.h"
struct AiCfg{
std::vector<int64_t> kind_inputs;
std::vector<int64_t> size_inputs;
std::vector<std::string> names;
std::vector<const char*> name_inputs;
//int m_sizeinput;
std::vector<std::vector<int64_t>> shape_inputs;
std::vector<int64_t> kind_outputs;
std::vector<int64_t> size_outputs;
std::vector<const char*> name_outputs;
//int m_sizeoutput;
std::vector<std::vector<int64_t>> shape_outputs;
const char** names_in;
const char** names_out;
void dump();
int inShape(int inx,int dim,int val);
int outShape(int inx,int dim,int val);
AiCfg clone();
};
class AiModel{
protected:
int m_inited;
std::string m_modelPath;
std::string m_modelbin;
std::string m_modelparam;
AiCfg *m_cfg;
virtual int doInitModel();
virtual int doRunModel(void** arrin,void** arrout,void* stream,AiCfg* pcfg=nullptr);
public:
void dump();
AiCfg config();
int pushName(const char* name,int input);
int initModel(std::string& modelpath);
int initModel(std::string& binfn,std::string& paramfn);
int runModel(void** arrin,void** arrout,void* stream,AiCfg* pcfg = nullptr);
AiModel();
virtual ~AiModel();
};
#define _ONNX_
#ifdef _ONNX_
#include "onnx/onnxruntime_cxx_api.h"
class OnnxModel:public AiModel{
protected:
int m_batch = 0;
int m_width = 640;
int m_height = 960;
Ort::Env env{nullptr};
Ort::SessionOptions sessionOptions{nullptr};
Ort::Session session{nullptr};
int doInitModel()override;
int doRunModel(void** arrin,void** arrout,void* stream,AiCfg* pcfg=nullptr)override;
public:
OnnxModel(int b,int w,int h);
OnnxModel();
virtual ~OnnxModel();
};
#endif
#define _NCNN_
#ifdef _NCNN_
#include "net.h"
class NcnnModel:public AiModel{
protected:
int m_width = 160;
int m_height = 160;
ncnn::Net net;
int doInitModel()override;
int doRunModel(void** arrin,void** arrout,void* stream,AiCfg* pcfg=nullptr)override;
public:
NcnnModel(int w,int h);
NcnnModel();
virtual ~NcnnModel();
};
#endif
#ifdef _TENSORRT_
#include "NvInfer.h"
#include "NvInferLegacyDims.h"
#include "NvInferRuntime.h"
class Logger:public nvinfer1::ILogger {
public:
void log(nvinfer1::ILogger::Severity severity, const char *msg) noexcept override {
// suppress info-level messages
if (severity == Severity::kINFO)
return;
switch (severity) {
case Severity::kINTERNAL_ERROR:
std::cerr << "INTERNAL_ERROR: ";
break;
case Severity::kERROR:
std::cerr << "ERROR: ";
break;
case Severity::kWARNING:
std::cerr << "WARNING: ";
break;
case Severity::kINFO:
std::cerr << "INFO: ";
break;
default:
std::cerr << "UNKNOWN: ";
break;
}
std::cerr << msg << std::endl;
}
};
class TrtModel:public AiModel{
protected:
Logger m_logger;
nvinfer1::IExecutionContext *m_context;
nvinfer1::ICudaEngine *m_engine;
int doInitModel()override;
int doRunModel(void** arrin,void** arrout,void* stream,AiCfg* pcfg=nullptr)override;
public:
TrtModel(int b,int w,int h);
TrtModel();
virtual ~TrtModel();
};
#endif

View File

@@ -0,0 +1,437 @@
#include <stdio.h>
#include <math.h>
#include <stdlib.h>
#include "blendgram.h"
void exColorBlend_Normal(uint8* T,uint8* A,uint8* B){ ColorBlend_Buffer(T,A,B,Normal); }
void exColorBlend_Lighten(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,Lighten);}
void exColorBlend_Darken(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,Darken);}
void exColorBlend_Multiply(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,Multiply);}
void exColorBlend_Average(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,Average);}
void exColorBlend_Add(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,Add);}
void exColorBlend_Subtract(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,Subtract);}
void exColorBlend_Difference(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,Difference);}
void exColorBlend_Negation(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,Negation);}
void exColorBlend_Screen(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,Screen);}
void exColorBlend_Exclusion(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,Exclusion);}
void exColorBlend_Overlay(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,Overlay);}
void exColorBlend_SoftLight(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,SoftLight);}
void exColorBlend_HardLight(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,HardLight);}
void exColorBlend_ColorDodge(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,ColorDodge);}
void exColorBlend_ColorBurn(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,ColorBurn);}
void exColorBlend_LinearDodge(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,LinearDodge);}
void exColorBlend_LinearBurn(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,LinearBurn);}
void exColorBlend_LinearLight(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,LinearLight);}
void exColorBlend_VividLight(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,VividLight);}
void exColorBlend_PinLight(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,PinLight);}
void exColorBlend_HardMix(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,HardMix);}
void exColorBlend_Reflect(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,Reflect);}
void exColorBlend_Glow(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,Glow);}
void exColorBlend_Phoenix(uint8* T,uint8* A,uint8* B) { ColorBlend_Buffer(T,A,B,Phoenix);}
typedef void (*BlendFunc) (uint8* T,uint8* A,uint8* B);
static int MAX_FUNC = 25;
static BlendFunc blendfuncs[25]={
&exColorBlend_Normal,
&exColorBlend_Lighten,
&exColorBlend_Darken,
&exColorBlend_Multiply,
&exColorBlend_Average,
&exColorBlend_Add,
&exColorBlend_Subtract,
&exColorBlend_Difference,
&exColorBlend_Negation,
&exColorBlend_Screen,
&exColorBlend_Exclusion,
&exColorBlend_Overlay,
&exColorBlend_SoftLight,
&exColorBlend_HardLight,
&exColorBlend_ColorDodge,
&exColorBlend_ColorBurn,
&exColorBlend_LinearDodge,
&exColorBlend_LinearBurn,
&exColorBlend_LinearLight,
&exColorBlend_VividLight,
&exColorBlend_PinLight,
&exColorBlend_HardMix,
&exColorBlend_Reflect,
&exColorBlend_Glow,
&exColorBlend_Phoenix
};
void BlendGramSimp(unsigned char *Src,unsigned char* Mask, unsigned char *Dest, int Width, int Height, int Mode)
{
if(Mode<1)return;
if(Mode>=MAX_FUNC)return;
BlendFunc func=blendfuncs[Mode];
unsigned char *LinePS, *LinePD,*LinePM;
for (int Y = 0; Y < Height; Y += 1)
{
LinePS = Src + Y * Width * 4;
LinePM = Mask + Y * Width * 4;
LinePD = Dest + Y * Width * 4;
for (int X = 0; X < Width; X += 1)
{
func(LinePD,LinePS,LinePM);
LinePS += 4;
LinePM += 4;
LinePD += 4;
}
}
}
void BlendGramAlpha3(unsigned char *Src,unsigned char* Mask, unsigned char *Dest, int Width, int Height)
{
printf("w %d h %d\n",Width,Height);
unsigned char *LinePS, *LinePD,*LinePM;
for (int Y = 0; Y < Height; Y += 1)
{
LinePS = Src + Y * Width * 3;
LinePM = Mask + Y * Width * 3;
LinePD = Dest + Y * Width * 3;
for (int X = 0; X < Width; X += 1)
{
//func(LinePD,LinePS,LinePM);
//ColorBlend_Alpha(LinePD,LinePD,LinePS,*LinePM);
float alpha = *LinePM/255.0f;
float beta = 1.0f-alpha;
//if(beta<0.5f) printf("==alpha %f beta %f\n",alpha,beta);
//if(beta<0.5f) printf("od %u ps %u\n",LinePD[0],LinePS[0]);
LinePD[0] = CLAMPCOLOR( LinePD[0]*alpha+LinePS[0]*beta);
//if(beta<0.5f) printf("new %u ps%u \n",LinePD[0],LinePS[0]);
//if(beta<0.5f) getchar();
LinePD[1] = CLAMPCOLOR(LinePD[1]*alpha+LinePS[1]*beta);
LinePD[2] = CLAMPCOLOR( LinePD[2]*alpha+LinePS[2]*beta);
LinePS += 3;
LinePM += 3;
LinePD += 3;
}
}
}
void BlendGramAlpha(unsigned char *Src,unsigned char* Mask, unsigned char *Dest, int Width, int Height)
{
unsigned char *LinePS, *LinePD,*LinePM;
for (int Y = 0; Y < Height; Y += 1)
{
LinePS = Src + Y * Width * 3;
LinePM = Mask + Y * Width * 1;
LinePD = Dest + Y * Width * 3;
for (int X = 0; X < Width; X += 1)
{
//func(LinePD,LinePS,LinePM);
ColorBlend_Alpha(LinePD,LinePD,LinePS,*LinePM);
/*
float alpha = *LinePM/255.0f;
float beta = 1.0f-alpha;
//printf("==alpha %f beta %f\n",alpha,beta);
LinePD[0] = LinePD[0]*alpha+LinePS[0]*beta;
LinePD[1] = LinePD[1]*alpha+LinePS[1]*beta;
LinePD[2] = LinePD[2]*alpha+LinePS[2]*beta;
*/
LinePS += 3;
LinePM += 1;
LinePD += 3;
}
}
}
void BlendGramAlphaRev(unsigned char *Src,unsigned char* Mask, unsigned char *Dest, int Width, int Height)
{
unsigned char *LinePS, *LinePD,*LinePM;
for (int Y = 0; Y < Height; Y += 1)
{
LinePS = Src + Y * Width * 3;
LinePM = Mask + Y * Width * 1;
LinePD = Dest + Y * Width * 3;
for (int X = 0; X < Width; X += 1)
{
//func(LinePD,LinePS,LinePM);
ColorBlend_Alpha(LinePD,LinePS,LinePD,*LinePM);
LinePS += 3;
LinePM += 1;
LinePD += 3;
}
}
}
/*
void BlendGram(CBitmap* image,CBitmap* mask,int mode)
{
if(mode<1)return;
if(mode>=MAX_FUNC)return;
BlendFunc func=blendfuncs[mode];
int Stride=image->width*4;
unsigned char *LinePS, *LinePD,*LinePM;
for (int Y = 0; Y < image->height; Y += 1)
{
LinePS = (unsigned char*)image->pixels +image->stride*Y;
LinePM = (unsigned char*)mask->pixels + mask->stride*Y;
LinePD = (unsigned char*)image->pixels +image->stride*Y;
for (int X = 0; X < image->width; X += 1)
{
func(LinePD,LinePS,LinePM);
LinePS += 4;
LinePM += 4;
LinePD += 4;
}
}
}
void BlendImageAdjustWithMask(CBitmap* bmp,CBitmap* adj,CBitmap* dst ,CBitmap* msk,int mode)
{
unsigned char* bmppixels=(unsigned char*)bmp->pixels;
unsigned char* mskpixels=(unsigned char*)msk->pixels;
unsigned char* dstpixels=(unsigned char*)dst->pixels;
unsigned char* adjpixels=(unsigned char*)adj->pixels;
int stride=bmp->stride;
int width=bmp->width;
int height=bmp->height;
int X,Y;
unsigned char* LinePS , * LinePM , * LinePD , * LinePA ;
#pragma omp parallel for private(LinePS,LinePM,LinePD,LinePA,X,Y)
for (Y = 0; Y < height; Y ++)
{
int offset=stride*Y;
LinePS = bmppixels +offset;
LinePM = mskpixels +offset;
LinePD = dstpixels +offset;
LinePA = adjpixels +offset;
for (X = 0; X < width; X ++)
{
unsigned char M=*LinePM;
if(M==0xFF){
LinePD[0]=LinePS[0];
LinePD[1]=LinePS[1];
LinePD[2]=LinePS[2];
}else if(M==0x00){
LinePD[0]=LinePA[0];
LinePD[1]=LinePA[1];
LinePD[2]=LinePA[2];
}else{
ColorBlend_Alpha(LinePD,LinePS,LinePA,M);
}
LinePD[3]=LinePS[3];
LinePS += 4; LinePM += 4; LinePD += 4; LinePA += 4;
}
}
}
void BlendImageAdjustWithMaskEx(CBitmap* bmp,CBitmap* adj,CBitmap* dst ,CBitmap* msk,int mode)
{
unsigned char* bmppixels=(unsigned char*)bmp->pixels;
unsigned char* mskpixels=(unsigned char*)msk->pixels;
unsigned char* dstpixels=(unsigned char*)dst->pixels;
unsigned char* adjpixels=(unsigned char*)adj->pixels;
int stride=bmp->stride;
int width=bmp->width;
int height=bmp->height;
int X,Y;
unsigned char* LinePS , * LinePM , * LinePD , * LinePA ;
#pragma omp parallel for private(LinePS,LinePM,LinePD,LinePA,X,Y)
for (Y = 0; Y < height; Y ++)
{
int offset=stride*Y;
LinePS = bmppixels +offset;
LinePM = mskpixels +offset;
LinePD = dstpixels +offset;
LinePA = adjpixels +offset;
for (X = 0; X < width; X ++)
{
unsigned char M=*LinePM;
if(M==0xFF){
LinePD[0]=LinePS[0];
LinePD[1]=LinePS[1];
LinePD[2]=LinePS[2];
}else if(M==0x00){
LinePD[0]=LinePA[0];
LinePD[1]=LinePA[1];
LinePD[2]=LinePA[2];
}else{
//ColorBlend_Alpha(LinePD,LinePS,LinePA,M);
LinePD[0]=LinePS[0]*M>>8;
LinePD[1]=LinePS[1]*M>>8;
LinePD[2]=LinePS[2]*M>>8;
}
LinePD[3]=M;
LinePS += 4; LinePM += 4; LinePD += 4; LinePA += 4;
}
}
}
void BlendImageAdjustWithAlpha(CBitmap* bmp,CBitmap* adj,CBitmap* dst ,int alpha,int mode){
unsigned char* bmppixels=(unsigned char*)bmp->pixels;
unsigned char* dstpixels=(unsigned char*)dst->pixels;
unsigned char* adjpixels=(unsigned char*)adj->pixels;
int stride=bmp->stride;
int width=bmp->width;
int height=bmp->height;
int X,Y;
unsigned char M=CLAMPCOLOR(alpha);
unsigned char *LinePS , *LinePD , *LinePA ;
#pragma omp parallel for private(LinePS,LinePD,LinePA,X,Y)
for (Y = 0; Y < height; Y ++)
{
int offset=stride*Y;
LinePS = bmppixels +offset;
LinePD = dstpixels +offset;
LinePA = adjpixels +offset;
for (X = 0; X < width; X ++)
{
if(M==0xFF){
LinePD[0]=LinePS[0];
LinePD[1]=LinePS[1];
LinePD[2]=LinePS[2];
}else if(M==0x00){
LinePD[0]=LinePA[0];
LinePD[1]=LinePA[1];
LinePD[2]=LinePA[2];
}else{
ColorBlend_Alpha(LinePD,LinePS,LinePA,M);
}
LinePD[3]=LinePS[3];
LinePS += 4; LinePD += 4; LinePA += 4;
}
}
}
void BlendImageAdjustWithAlphaMask(CBitmap* bmp,CBitmap* adj,CBitmap* dst ,CBitmap* msk,int alpha,int mode){
unsigned char* bmppixels=(unsigned char*)bmp->pixels;
unsigned char* mskpixels=(unsigned char*)msk->pixels;
unsigned char* dstpixels=(unsigned char*)dst->pixels;
unsigned char* adjpixels=(unsigned char*)adj->pixels;
int stride=bmp->stride;
int width=bmp->width;
int height=bmp->height;
int X,Y;
unsigned char NM=CLAMPCOLOR(alpha);
unsigned char *LinePS , *LinePM , *LinePD , *LinePA ;
#pragma omp parallel for private(LinePS,LinePM,LinePD,LinePA,X,Y)
for (Y = 0; Y < height; Y ++)
{
int offset=stride*Y;
LinePS = bmppixels +offset;
LinePM = mskpixels +offset;
LinePD = dstpixels +offset;
LinePA = adjpixels +offset;
for (X = 0; X < width; X ++)
{
unsigned char M=*LinePM;
if(M==0xFF){
LinePD[0]=LinePS[0];
LinePD[1]=LinePS[1];
LinePD[2]=LinePS[2];
}else if(M==0x00){
if(NM==0xFF){
LinePD[0]=LinePS[0];
LinePD[1]=LinePS[1];
LinePD[2]=LinePS[2];
}else {
if(NM==0x00){
//none
LinePD[0]=LinePA[0];
LinePD[1]=LinePA[1];
LinePD[2]=LinePA[2];
}else{
ColorBlend_Alpha(LinePD,LinePS,LinePA,NM);
}
}
}else{
//
if(NM==0xFF){
LinePD[0]=LinePS[0];
LinePD[1]=LinePS[1];
LinePD[2]=LinePS[2];
}else{
if(NM==0x00){
ColorBlend_Alpha(LinePD,LinePS,LinePA,M);
}else{
ColorBlend_Alpha(LinePA,LinePS,LinePA,NM);
ColorBlend_Alpha(LinePD,LinePS,LinePA,M);
}
}
}
LinePD[3]=LinePS[3];
LinePS += 4; LinePM += 4; LinePD += 4; LinePA += 4;
}
}
}
void ReadAlphaBySrc(CBitmap* src,CBitmap* alpha){
memcpy(alpha,src,sizeof(CBitmap));
alpha->stride=src->width;
alpha->channel=1;
alpha->pixels=(CPixel*)malloc(alpha->width*alpha->height*sizeof(unsigned char));
unsigned char* bmppixels=(unsigned char*)src->pixels;
unsigned char* alapixels=(unsigned char*)alpha->pixels;
int stride=src->stride;
int width=src->width;
int height=src->height;
int X,Y;
unsigned char *LinePS , *LinePA;
//#pragma omp parallel for private(LinePS,LinePA)
for (Y = 0; Y < height; Y ++)
{
LinePS = bmppixels +stride*Y;
LinePA = alapixels +width*Y;
for (X = 0; X < width; X ++)
{
LinePA[0]=LinePS[3];
LinePS += 4; LinePA ++;
}
}
}
void CheckAlpha(CBitmap* bmp,CBitmap* alpha)
{
unsigned char* bmppixels=(unsigned char*)bmp->pixels;
unsigned char* alapixels=(unsigned char*)alpha->pixels;
int stride=bmp->stride;
int width=bmp->width;
int height=bmp->height;
int X,Y;
unsigned char *LinePS , *LinePA;
//#pragma omp parallel for private(LinePS,LinePA)
for (Y = 0; Y < height; Y ++)
{
LinePS = bmppixels +stride*Y;
LinePA = alapixels +width*Y;
for (X = 0; X < width; X ++)
{
//unsigned char M=LinePA[0];
if(*LinePA==0x00){
LinePS[0]=0;
LinePS[1]=0;
LinePS[2]=0;
LinePS[3]=0;
//}else if(M<0xff){
//if(LinePD[0]>M)LinePD[0]=M;
//if(LinePD[1]>M)LinePD[1]=M;
//if(LinePD[2]>M)LinePD[2]=M;
//LinePD[3]=M;
}else{
}
LinePS += 4; LinePA++;
}
}
}
*/

View File

@@ -0,0 +1,287 @@
#ifndef __BLENDGRAM_H__
#define __BLENDGRAM_H__
#include <stdlib.h>
#include <stdio.h>
#include <math.h>
typedef unsigned char uchar;
#define CLAMPCOLOR(x) (uchar)((x)<(0)?(0):((x)>(255)?(255):(x)))
#define MMAX(A,B) ((A)>(B)?(A):(B))
#define MMIN(A,B) ((A)<(B)?(A):(B))
static int ConstBlend_Buffer = 0;
static int ConstBlend_Normal=ConstBlend_Buffer+1;
static int ConstBlend_Lighten=ConstBlend_Buffer+2;
static int ConstBlend_Darken=ConstBlend_Buffer+3;
static int ConstBlend_Multiply=ConstBlend_Buffer+4;
static int ConstBlend_Average=ConstBlend_Buffer+5;
static int ConstBlend_Add=ConstBlend_Buffer+6;
static int ConstBlend_Subtract=ConstBlend_Buffer+7;
static int ConstBlend_Difference=ConstBlend_Buffer+8;
static int ConstBlend_Negation=ConstBlend_Buffer+9;
static int ConstBlend_Screen=ConstBlend_Buffer+10;
static int ConstBlend_Exclusion=ConstBlend_Buffer+11;
static int ConstBlend_Overlay=ConstBlend_Buffer+12;
static int ConstBlend_SoftLight=ConstBlend_Buffer+13;
static int ConstBlend_HardLight=ConstBlend_Buffer+14;
static int ConstBlend_ColorDodge=ConstBlend_Buffer+15;
static int ConstBlend_ColorBurn=ConstBlend_Buffer+16;
static int ConstBlend_LinearDodge=ConstBlend_Buffer+17;
static int ConstBlend_LinearBurn=ConstBlend_Buffer+18;
static int ConstBlend_LinearLight=ConstBlend_Buffer+19;
static int ConstBlend_VividLight=ConstBlend_Buffer+20;
static int ConstBlend_PinLight=ConstBlend_Buffer+21;
static int ConstBlend_HardMix=ConstBlend_Buffer+22;
static int ConstBlend_Reflect=ConstBlend_Buffer+23;
static int ConstBlend_Glow=ConstBlend_Buffer+24;
static int ConstBlend_Phoenix=ConstBlend_Buffer+25;
//void BlendGram(CBitmap* immage,CBitmap* mask,int mode);
//#typedef unsigned char uint8
#define uint8 unsigned char
#define float64 double
#define TRUE 1
#define FALSE 0
inline uint8 mmin(uint8 A,uint8 B){
return A<B?A:B;
}
inline uint8 mmax(uint8 A,uint8 B){
return A>B?A:B;
}
#define ChannelBlend_Normal(A,B) ((uint8)(A))
#define ChannelBlend_Lighten(A,B) ((uint8)((B > A) ? B:A))
#define ChannelBlend_Darken(A,B) ((uint8)((B > A) ? A:B))
#define ChannelBlend_Multiply(A,B) ((uint8)((A * B) / 255))
#define ChannelBlend_Average(A,B) ((uint8)((A + B) / 2))
#define ChannelBlend_Add(A,B) ((uint8)(mmin(255, (A + B))))
#define ChannelBlend_Subtract(A,B) ((uint8)((A + B < 255) ? 0:(A + B - 255)))
#define ChannelBlend_Difference(A,B) ((uint8)(abs(A - B)))
#define ChannelBlend_Negation(A,B) ((uint8)(255 - abs(255 - A - B)))
#define ChannelBlend_Screen(A,B) ((uint8)(255 - (((255 - A) * (255 - B)) >> 8)))
#define ChannelBlend_Exclusion(A,B) ((uint8)(A + B - 2 * A * B / 255))
#define ChannelBlend_Overlay(A,B) ((uint8)((B < 128) ? (2 * A * B / 255):(255 - 2 * (255 - A) * (255 - B) / 255)))
#define ChannelBlend_SoftLight(A,B) ((uint8)((B < 128)?(2*((A>>1)+64))*((float)B/255):(255-(2*(255-((A>>1)+64))*(float)(255-B)/255))))
#define ChannelBlend_HardLight(A,B) (ChannelBlend_Overlay(B,A))
#define ChannelBlend_ColorDodge(A,B) ((uint8)((B == 255) ? B:mmin(255, ((A << 8 ) / (255 - B)))))
#define ChannelBlend_ColorBurn(A,B) ((uint8)((B == 0) ? B:mmax(0, (255 - ((255 - A) << 8 ) / B))))
#define ChannelBlend_LinearDodge(A,B)(ChannelBlend_Add(A,B))
#define ChannelBlend_LinearBurn(A,B) (ChannelBlend_Subtract(A,B))
#define ChannelBlend_LinearLight(A,B)((uint8)(B < 128)?ChannelBlend_LinearBurn(A,(2 * B)):ChannelBlend_LinearDodge(A,(2 * (B - 128))))
#define ChannelBlend_VividLight(A,B) ((uint8)(B < 128)?ChannelBlend_ColorBurn(A,(2 * B)):ChannelBlend_ColorDodge(A,(2 * (B - 128))))
#define ChannelBlend_PinLight(A,B) ((uint8)(B < 128)?ChannelBlend_Darken(A,(2 * B)):ChannelBlend_Lighten(A,(2 * (B - 128))))
#define ChannelBlend_HardMix(A,B) ((uint8)((ChannelBlend_VividLight(A,B) < 128) ? 0:255))
#define ChannelBlend_Reflect(A,B) ((uint8)((B == 255) ? B:mmin(255, (A * A / (255 - B)))))
#define ChannelBlend_Glow(A,B) (ChannelBlend_Reflect(B,A))
#define ChannelBlend_Phoenix(A,B) ((uint8)(mmin(A,B) - mmax(A,B) + 255))
#define ChannelBlend_SoftEx(A,B) (A*B/255+A*(255-((255-A)*(255-B)/255)-A*B/255)/255)
#define ChannelBlend_Alpha(A,B,O) ((uint8)(O * A + (1 - O) * B))
#define ChannelBlend_AlphaEx(A,B,O) ((uint8)((O * A + (255 - O) * B)/255))
#define ChannelBlend_AlphaF(A,B,F,O) (ChannelBlend_AlphaEx(F(A,B),A,O))
#define ColorBlend_Alpha(T,A,B,O) (T)[0] = ChannelBlend_AlphaEx((A)[0], (B)[0],O), (T)[1] = ChannelBlend_AlphaEx((A)[1], (B)[1],O), (T)[2] = ChannelBlend_AlphaEx((A)[2], (B)[2],O)
//, (T)[3] = ChannelBlend_AlphaEx((A)[3], (B)[3],O)
#define ColorBlend_AlphaF(T,A,B,F,O) (T)[0] = ChannelBlend_AlphaF((A)[0], (B)[0],F,O), (T)[1] = ChannelBlend_AlphaF((A)[1], (B)[1],F,O), (T)[2] = ChannelBlend_AlphaF((A)[2], (B )[2],F,O) , (T)[3] = ChannelBlend_AlphaEx((A)[3], (B)[3],O)
#define ColorBlend_Buffer(T,A,B,M) (T)[0] = ChannelBlend_##M((A)[0], (B)[0]), (T)[1] = ChannelBlend_##M((A)[1], (B)[1]), (T)[2] = ChannelBlend_##M((A)[2], (B)[2])
#define ColorBlend_Normal(T,A,B) (ColorBlend_Buffer(T,A,B,Normal))
#define ColorBlend_Lighten(T,A,B) (ColorBlend_Buffer(T,A,B,Lighten))
#define ColorBlend_Darken(T,A,B) (ColorBlend_Buffer(T,A,B,Darken))
#define ColorBlend_Multiply(T,A,B) (ColorBlend_Buffer(T,A,B,Multiply))
#define ColorBlend_Average(T,A,B) (ColorBlend_Buffer(T,A,B,Average))
#define ColorBlend_Add(T,A,B) (ColorBlend_Buffer(T,A,B,Add))
#define ColorBlend_Subtract(T,A,B) (ColorBlend_Buffer(T,A,B,Subtract))
#define ColorBlend_Difference(T,A,B) (ColorBlend_Buffer(T,A,B,Difference))
#define ColorBlend_Negation(T,A,B) (ColorBlend_Buffer(T,A,B,Negation))
#define ColorBlend_Screen(T,A,B) (ColorBlend_Buffer(T,A,B,Screen))
#define ColorBlend_Exclusion(T,A,B) (ColorBlend_Buffer(T,A,B,Exclusion))
#define ColorBlend_Overlay(T,A,B) (ColorBlend_Buffer(T,A,B,Overlay))
#define ColorBlend_SoftLight(T,A,B) (ColorBlend_Buffer(T,A,B,SoftLight))
#define ColorBlend_HardLight(T,A,B) (ColorBlend_Buffer(T,A,B,HardLight))
#define ColorBlend_ColorDodge(T,A,B) (ColorBlend_Buffer(T,A,B,ColorDodge))
#define ColorBlend_ColorBurn(T,A,B) (ColorBlend_Buffer(T,A,B,ColorBurn))
#define ColorBlend_LinearDodge(T,A,B) (ColorBlend_Buffer(T,A,B,LinearDodge))
#define ColorBlend_LinearBurn(T,A,B) (ColorBlend_Buffer(T,A,B,LinearBurn))
#define ColorBlend_LinearLight(T,A,B) (ColorBlend_Buffer(T,A,B,LinearLight))
#define ColorBlend_VividLight(T,A,B) (ColorBlend_Buffer(T,A,B,VividLight))
#define ColorBlend_PinLight(T,A,B) (ColorBlend_Buffer(T,A,B,PinLight))
#define ColorBlend_HardMix(T,A,B) (ColorBlend_Buffer(T,A,B,HardMix))
#define ColorBlend_Reflect(T,A,B) (ColorBlend_Buffer(T,A,B,Reflect))
#define ColorBlend_Glow(T,A,B) (ColorBlend_Buffer(T,A,B,Glow))
#define ColorBlend_Phoenix(T,A,B) (ColorBlend_Buffer(T,A,B,Phoenix))
#define ColorBlend_Hue(T,B,L) ColorBlend_Hls(T,B,L,HueL,LuminationB,SaturationB)
#define ColorBlend_Saturation(T,B,L) ColorBlend_Hls(T,B,L,HueB,LuminationB,SaturationL)
#define ColorBlend_Color(T,B,L) ColorBlend_Hls(T,B,L,HueL,LuminationB,SaturationL)
#define ColorBlend_Luminosity(T,B,L) ColorBlend_Hls(T,B,L,HueB,LuminationL,SaturationB)
#define ColorBlend_Hls(T,B,L,O1,O2,O3) { \
float64 HueB, LuminationB, SaturationB; \
float64 HueL, LuminationL, SaturationL; \
Color_RgbToHls((B)[2],(B)[1],(B)[0], &HueB, &LuminationB, &SaturationB); \
Color_RgbToHls((L)[2],(L)[1],(L)[0], &HueL, &LuminationL, &SaturationL); \
Color_HlsToRgb(O1,O2,O3,&(T)[2],&(T)[1],&(T)[0]); \
}
/*********************************************************************/
#define COLOR_OPAQUE (0)
#define COLOR_TRANSPARENT (127)
#define RGB_SIZE (3)
#define RGB_BPP (24)
#define RGB_MAXRED (255)
#define RGB_MAXGREEN (255)
#define RGB_MAXBLUE (255)
#define ARGB_SIZE (4)
#define ARGB_BPP (32)
#define ARGB_MAXALPHA (127)
#define ARGB_MAXRED (RGB_MAXRED)
#define ARGB_MAXGREEN (RGB_MAXGREEN)
#define ARGB_MAXBLUE (RGB_MAXBLUE)
/*********************************************************************/
#define Color_GetChannel(c,shift) ((uint8)((c) >> (shift)))
#define Color_Reverse(c,bpp) ((((uint8)(c) << 24) | ((uint8)((c) >> 8 ) << 16) | ((uint8)((c) >> 16) << 8 ) | \ ((uint8)((c) >> 24))) >> (32 - (bpp)))
#define Rgb_ByteWidth(width) ((width) * RGB_SIZE)
#define Rgb_PixelWidth(width) ((width) / RGB_SIZE)
#define Rgb_GetRed(rgb) (Color_GetChannel(rgb, 0))
#define Rgb_GetGreen(rgb) (Color_GetChannel(rgb, 8))
#define Rgb_GetBlue(rgb) (Color_GetChannel(rgb, 16))
#define Rgba_GetRed(rgba) (Color_GetChannel(rgba, 24))
#define Rgba_GetGreen(rgba) (Color_GetChannel(rgba, 16))
#define Rgba_GetBlue(rgba) (Color_GetChannel(rgba, 8))
#define Rgba_GetAlpha(rgba) (Color_GetChannel(rgba, 0))
#define Argb_GetAlpha(argb) (Color_GetChannel(argb, 24))
#define Argb_GetRed(argb) (Color_GetChannel(argb, 16))
#define Argb_GetGreen(argb) (Color_GetChannel(argb, 8))
#define Argb_GetBlue(argb) (Color_GetChannel(argb, 0))
#define MakeRgb(r,g,b) (((uint32)(uint8)(b) << 16) | ((uint16)(uint8)(g) << 8 ) | (uint8)(r))
#define MakeRgba(r,g,b,a) (((uint32)(uint8)(r) << 24) | ((uint16)(uint8)(g) << 16) | ((uint16)(uint8)(b) << 8 ) | (uint8)(a))
#define MakeArgb(a,r,g,b) (((uint32)(uint8)(a) << 24) | ((uint32)(uint8)(r) << 16) | ((uint16)(uint8)(g) << 8 ) | (uint8)(b))
#define HexToRgb(hex) (MakeRgb(((hex & 0xFF0000) >> 16), ((hex & 0x00FF00) >> 8 ), (hex & 0xFF)))
inline int Color_HueToRgb(float64 M1, float64 M2, float64 Hue, float64 *Channel)
{
if (Hue < 0.0)
Hue += 1.0;
else if (Hue > 1.0)
Hue -= 1.0;
if ((6.0 * Hue) < 1.0)
*Channel = (M1 + (M2 - M1) * Hue * 6.0);
else if ((2.0 * Hue) < 1.0)
*Channel = (M2);
else if ((3.0 * Hue) < 2.0)
*Channel = (M1 + (M2 - M1) * ((2.0F / 3.0F) - Hue) * 6.0);
else
*Channel = (M1);
return TRUE;
}
inline void Color_RgbToHls(uint8 Red, uint8 Green, uint8 Blue, float64 *Hue, float64 *Lumination, float64 *Saturation)
{
float64 Delta;
float64 Max, Min;
float64 Redf, Greenf, Bluef;
Redf = (float64)Red / 255.0;
Greenf = (float64)Green / 255.0;
Bluef = (float64)Blue / 255.0;
//Max = fmax(fmax(Redf, Greenf), Bluef);
//Min = fmin(fmin(Redf, Greenf), Bluef);
Max = MMAX(MMAX(Red, Green), Blue)/255.0;
Min = MMIN(MMIN(Red, Green), Blue)/255.0;
*Hue = 0;
*Lumination = (Max + Min) / 2.0F;
*Saturation = 0;
if (Max == Min)
return ;
Delta = (Max - Min);
if (*Lumination < 0.5)
*Saturation = Delta / (Max + Min);
else
*Saturation = Delta / (2.0 - Max - Min);
if (Redf == Max)
*Hue = (Greenf - Bluef) / Delta;
else if (Greenf == Max)
*Hue = 2.0 + (Bluef - Redf) / Delta;
else
*Hue = 4.0 + (Redf - Greenf) / Delta;
*Hue /= 6.0;
if (*Hue < 0.0)
*Hue += 1.0;
}
inline void Color_HlsToRgb(float64 Hue, float64 Lumination, float64 Saturation, uint8 *Red, uint8 *Green, uint8 *Blue)
{
float64 M1, M2;
float64 Redf, Greenf, Bluef;
if (Saturation == 0) {
Redf = Lumination;
Greenf = Lumination;
Bluef = Lumination;
} else {
if (Lumination <= 0.5)
M2 = Lumination * (1.0 + Saturation);
else
M2 = Lumination + Saturation - Lumination * Saturation;
M1 = (2.0 * Lumination - M2);
Color_HueToRgb(M1, M2, Hue + (1.0F / 3.0F), &Redf);
Color_HueToRgb(M1, M2, Hue, &Greenf);
Color_HueToRgb(M1, M2, Hue - (1.0F / 3.0F), &Bluef);
}
*Red = (uint8)(Redf * 255);
*Blue = (uint8)(Bluef * 255);
*Green = (uint8)(Greenf * 255);
}
void BlendGramSimp(unsigned char *Src,unsigned char* Mask, unsigned char *Dest, int Width, int Height, int Mode);
void BlendGramAlpha(unsigned char *Src,unsigned char* Mask, unsigned char *Dest, int Width, int Height);
void BlendGramAlpha3(unsigned char *Src,unsigned char* Mask, unsigned char *Dest, int Width, int Height);
void BlendGramAlphaRev(unsigned char *Src,unsigned char* Mask, unsigned char *Dest, int Width, int Height);
/*
void BlendImageAdjustWithMask(CBitmap* bmp,CBitmap* adj,CBitmap* dst ,CBitmap* msk,int mode);
void BlendImageAdjustWithMaskEx(CBitmap* bmp,CBitmap* adj,CBitmap* dst ,CBitmap* msk,int mode);
void BlendImageAdjustWithAlpha(CBitmap* bmp,CBitmap* adj,CBitmap* dst ,int alpha,int mode);
void BlendImageAdjustWithAlphaMask(CBitmap* bmp,CBitmap* adj,CBitmap* dst ,CBitmap* msk,int alpha,int mode);
void CheckAlpha(CBitmap* bmp,CBitmap* alpha);
void ReadAlphaBySrc(CBitmap* src,CBitmap* alpha);
*/
#endif

View File

@@ -0,0 +1,19 @@
g++ -g -I. \
-I/usr/include/opencv4 \
-I../third/x86/include \
-I../third/x86/include/ncnn/ \
-I../third/x86/include/onnx/ \
-I../third/x86/include/turbojpeg/ \
wavcache.cpp \
jmat.cpp \
wavreader.cpp \
wenet.cpp \
aimodel.cpp \
scrfd.cpp \
pfpld.cpp \
munet.cpp \
blendgram.cpp \
face_utils.cpp \
-L../third/x86/lib \
-lopencv_core -lopencv_dnn -lopencv_imgcodecs -lopencv_imgproc -lopencv_highgui -lopencv_videoio \
-lonnxruntime -lncnn \

View File

@@ -0,0 +1,114 @@
#include "face_utils.h"
//#include <sys/timeb.h>
cv::Mat resize_image(cv::Mat srcimg, int height, int width, int* top, int* left){
cv::Mat dstimg;
int srch = srcimg.rows, srcw = srcimg.cols;
int neww = width;
int newh = height;
if (srch != srcw) {
float hw_scale = (float)srch / srcw;
if (hw_scale > 1) {
newh = height;
neww = int(width / hw_scale);
cv::resize(srcimg, dstimg, cv::Size(neww, newh), cv::INTER_AREA);
*left = int((width - neww) * 0.5);
cv::copyMakeBorder(dstimg, dstimg, 0, 0, *left, width - neww - *left, cv::BORDER_CONSTANT, 0);
}
else
{
newh = (int)height * hw_scale;
neww = width;
cv::resize(srcimg, dstimg,cv::Size(neww, newh), cv::INTER_AREA);
*top = (int)(height - newh) * 0.5;
cv::copyMakeBorder(dstimg, dstimg, *top, height - newh - *top, 0, 0, cv::BORDER_CONSTANT, 0);
}
} else {
cv::resize(srcimg, dstimg, cv::Size(neww, newh), cv::INTER_AREA);
}
return dstimg;
}
int dumpfile(char* file,char** pbuf){
std::string fname(file);
std::ifstream cache(fname,std::ios::binary);
cache.seekg(0,std::ios::end);
const int engSize = cache.tellg();
cache.seekg(0,std::ios::beg);
char *modelMem = (char*)malloc(engSize+8000);
cache.read(modelMem,engSize);
cache.close();
*pbuf = modelMem;
return engSize;
}
void dumpchar(char* abuf,int len){
uint8_t* buf = (uint8_t*)abuf;
printf("\n----------------------chardump------------------------\n");
int i;
for(i = 0; i < len; i++) {
printf("=%u=", buf[i]);
if( (i+1) % 16 == 0) {
printf("\n");
}
}
if(i%16 != 0) {
printf("\n");
}
printf("\n----------------------chardump------------------------\n");
}
void dumpfloat(float* abuf,int len){
printf("\n----------------------floatdump------------------------\n");
int i;
for(i = 0; i < len; i++) {
printf("=%f=", abuf[i]);
if( (i+1) % 16 == 0) {
printf("\n");
}
}
if(i%16 != 0) {
printf("\n");
}
printf("\n----------------------floatdump------------------------\n");
}
void dumphex(char* abuf,int len){
unsigned char* buf = (unsigned char*)abuf;
int i = 0;
printf("\n----------------------hexdump------------------------\n");
for(i = 0; i < len; i++) {
printf("=%02x=", buf[i]);
if( (i+1) % 16 == 0) {
printf("\n");
}
}
if(i%16 != 0) {
printf("\n");
}
printf("---------------------hexdump-------------------------\n\n");
}
int diffbuf(char* abuf,char* bbuf,int size){
char* pa = abuf;
char* pb = bbuf;
int diff = 0;
for(int k= 0;k<size;k++){
if(*pa++==*pb++){
}else{
diff++;
}
}
return diff;
}
uint64_t timer_msstamp() {
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (ts.tv_sec*1000l) + (ts.tv_nsec/CLOCKS_PER_SEC);
}

View File

@@ -0,0 +1,20 @@
#pragma once
#include <fstream>
#include <sstream>
#include <iostream>
#include <vector>
//#include <opencv2/dnn.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/highgui.hpp>
cv::Mat resize_image(cv::Mat srcimg, int height, int width, int* top, int* left);
void dumpchar(char* abuf,int len);
void dumphex(char* abuf,int len);
void dumpfloat(float* abuf,int len);
void dumpdouble(double* abuf,int len);
int dumpfile(char* file,char** pbuf);
int diffbuf(char* abuf,char* bbuf,int size);
uint64_t timer_msstamp();

View File

@@ -0,0 +1,31 @@
#pragma once
#include <stdio.h>
#ifdef __cplusplus
extern "C" {
#endif
#ifdef LIB_JNI
#include <jni.h>
#include <android/log.h>
#define LOG_TAG "tooken"
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__)
#define LOGE(...) __android_log_print(ANDROID_LOG_DEBUG,LOG_TAG,__VA_ARGS__)
#define LOGD(...) __android_log_print(ANDROID_LOG_DEBUG,LOG_TAG,__VA_ARGS__)
#else
#define LOGE(...) printf(__VA_ARGS__)
#define LOGI(...) printf(__VA_ARGS__)
#define LOGD(...) printf(__VA_ARGS__)
#endif
#ifdef __cplusplus
}
#endif

View File

@@ -0,0 +1,411 @@
#include "jmat.h"
extern "C"{
#pragma pack(push)
#pragma pack(4)
typedef struct _gpg_hdr {
char head[4];
int box[4];
int size[4];
int width[4];
int height[4];
uint8_t channel[4];
uint8_t bit[4];
}gpg_hdr;
#pragma pack(pop)
}
int JBuf::zeros(){
memset(m_buf,0,m_size);
return m_size;
}
int JBuf::forceref(int bref){
if(m_ref!=bref){
m_ref = bref;
}
return 0;
}
JBuf::JBuf(uint32_t size,void* buf ){
if(buf){
m_ref = true;
m_buf = buf;
m_size = size;
}else{
m_ref = false;
m_size = size;
m_buf = malloc(size+1024);
}
}
JBuf::~JBuf(){
if(!m_ref){
free(m_buf);
m_buf = nullptr;
}
}
JBuf::JBuf(){
m_size = 0;
m_buf = nullptr;
}
JMat::JMat(){
init_tagarr();
}
void JMat::init_tagarr(){
memset(m_tagarr,0,512*sizeof(int));
}
int* JMat::tagarr(){
return m_tagarr;
}
int JMat::savegpg(std::string gpgfile){
gpg_hdr ghead;
memset(&ghead,0,sizeof(gpg_hdr));
ghead.head[0]='g';
ghead.head[1]='p';
ghead.head[2]='g';
ghead.head[3]='1';
ghead.size[0]=m_size;
ghead.width[0]=m_width;
ghead.height[0]=m_height;
ghead.channel[0]=m_channel;
ghead.bit[0]=m_bit;
FILE *gpgFile = NULL;
const char* fn = gpgfile.c_str();
if ((gpgFile = fopen(fn, "wb")) == NULL)return -1;
fwrite(&ghead,sizeof(gpg_hdr),1,gpgFile);
fwrite(m_buf, m_size, 1, gpgFile);
fclose(gpgFile);
return 0;
}
int JMat::load(std::string picfile){
const char* fn = picfile.c_str();
int len = strlen(fn);
if(len<4)return -1;
fn+= len-3;
int gpg = (fn[0]=='g')&&(fn[1]=='p')&&(fn[2]=='g');
if(gpg){
return loadgpg(picfile);
}else{
return loadjpg(picfile);
}
}
int JMat::loadgpg(std::string gpgfile){
FILE *gpgFile = NULL;
const char* fn = gpgfile.c_str();
if ((gpgFile = fopen(fn, "rb")) == NULL)return -1;
int rst = 0;
while(1){
gpg_hdr ghead;
memset(&ghead,0,sizeof(gpg_hdr));
fread(&ghead,sizeof(gpg_hdr),1,gpgFile);
char* arr=ghead.head;
if((arr[0]=='g')&&
(arr[1]=='p')&&
(arr[2]=='g')){
int imgSize = ghead.size[0];
if(m_size<imgSize){
if((!m_ref)&&m_buf)free(m_buf);
m_buf = malloc(imgSize);
}
m_size = imgSize;
m_width = ghead.width[0];
m_height = ghead.height[0];
m_channel = ghead.channel[0];
m_bit = ghead.bit[0];
fread(m_buf, m_size, 1, gpgFile);
}else{
rst = -11;
}
break;
}
fclose(gpgFile);
return rst;
}
#ifdef USE_TURBOJPG
#include "turbojpeg.h"
int JMat::loadjpg(std::string picfile,int flag){
tjhandle tjInstance = NULL;
int rst = 0;
size_t jpegSize = 0;
size_t imgSize = 0;
int newbuf = 0;
unsigned char *jpegBuf = NULL;
if(1){
long size;
FILE *jpegFile = NULL;
const char* fn = picfile.c_str();
if ((jpegFile = fopen(fn, "rb")) == NULL)return -1;
if (fseek(jpegFile, 0, SEEK_END) < 0 || ((size = ftell(jpegFile)) < 0) || (fseek(jpegFile, 0, SEEK_SET) < 0)){
fclose(jpegFile);
return -2;
}
if (size == 0){
fclose(jpegFile);
return -3;
}
jpegSize = size;
jpegBuf = (unsigned char*)tj3Alloc(jpegSize);
fread(jpegBuf, jpegSize, 1, jpegFile);
fclose(jpegFile);
}
if ((tjInstance = tj3Init(TJINIT_DECOMPRESS)) == NULL)return -11;
while(1){
unsigned char *imgBuf = NULL;
int w, h;
int inSubsamp, inColorspace;
int pixelFormat = TJPF_BGR;
rst = tj3DecompressHeader(tjInstance, jpegBuf, jpegSize);
if(rst<0){
rst = -12;
break;
}
w = tj3Get(tjInstance, TJPARAM_JPEGWIDTH);
h = tj3Get(tjInstance, TJPARAM_JPEGHEIGHT);
inSubsamp = tj3Get(tjInstance, TJPARAM_SUBSAMP);
inColorspace = tj3Get(tjInstance, TJPARAM_COLORSPACE);
imgSize = w * h * tjPixelSize[pixelFormat];
if(imgSize <0){
rst = -13;
break;
}
//printf("===imgSize %d m_size %d\n",imgSize,m_size);
if(m_size<imgSize){
if((!m_ref)&&m_buf)free(m_buf);
m_buf = malloc(imgSize);
m_ref = 0;
}
m_size = imgSize;
imgBuf = (unsigned char *)m_buf;
if(tj3Decompress8(tjInstance, jpegBuf, jpegSize, imgBuf, 0, pixelFormat) < 0){
rst = -15;
break;
}
//m_ref = 0;
m_bit = 1;
m_channel = 3;
m_stride = w*3;
m_width = w;
m_height = h;
break;
}
if(jpegBuf)tj3Free(jpegBuf);
jpegBuf = NULL;
tj3Destroy(tjInstance);
tjInstance = NULL;
return rst;
}
#else
int JMat::loadjpg(std::string picfile,int flag){
return -1;
}
#endif
JMat::JMat(int w,int h,float *buf ,int c ,int d ):JBuf(){
m_bit = sizeof(float);
m_width = w;
m_height = h;
m_channel = c;
m_stride = d?d:w*c;
m_size = m_bit*m_stride*m_height;
m_buf = buf;
m_ref = 1;
init_tagarr();
}
JMat::JMat(int w,int h,uint8_t *buf ,int c ,int d ):JBuf(){
m_bit = 1;
m_width = w;
m_height = h;
m_channel = c;
m_stride = d?d:w*c;
m_size = m_bit*m_stride*m_height;
m_buf = buf;
m_ref = 1;
init_tagarr();
}
JMat::JMat(int w,int h,int c ,int d ,int b):JBuf(){
m_bit = b==0?sizeof(float):b;
m_width = w;
m_height = h;
m_channel = c;
m_stride = d?d:w*c;
m_size = m_bit*m_stride*m_height;
//printf("===mat %d size %d\n",m_bit,m_size);
m_buf = malloc(m_size+m_bit*m_stride);
memset(m_buf,0,m_size+m_bit*m_stride);
m_ref = 0;
init_tagarr();
}
#ifdef USE_OPENCV
cv::Mat JMat::cvmat(){
if(m_channel == 3){
cv::Mat rrr(m_height,m_width,m_bit==1?CV_8UC3:CV_32FC3,m_buf);
return rrr;
}else if(m_channel == 1){
cv::Mat rrr(m_height,m_width,m_bit==1?CV_8UC1:CV_32FC1,m_buf);
return rrr;
}else{
cv::Mat rrr(m_height,m_width*m_channel,m_bit==1?CV_8UC1:CV_32FC1,m_buf);
return rrr;
}
}
int JMat::show(const char* title){
std::string name(title);
cv::Mat mat(m_height,m_width,m_channel==3?CV_8UC3:CV_8UC1,m_buf);
cv::imshow(name,mat);
return 0;
}
int JMat::tojpg(const char* fn){
cv::Mat mat(m_height,m_width,CV_8UC3,m_buf);
std::string name(fn);
return cv::imwrite(name,mat);
}
#else
int JMat::show(const char* title){
return 0;
}
int JMat::tojpg(const char* fn){
return 0;
}
#endif
int JMat::tobin(const char* fn){
FILE* file = fopen(fn, "w");
if(!file)return 0;
fwrite(m_buf, m_size, 1, file);
fclose(file);
return 1;
}
JMat* JMat::refclone(int ref){
if(ref){
if(m_bit==1){
return new JMat(m_width,m_height,(uint8_t*)m_buf,m_channel,m_stride);
}else{
return new JMat(m_width,m_height,(float*)m_buf,m_channel,m_stride);
}
}else{
JMat* cm = new JMat(m_width,m_height,m_channel,m_stride,m_bit);
memcpy(cm->m_buf,m_buf,m_size);
memcpy(cm->m_tagarr,m_tagarr,512*sizeof(int));
return cm;
}
}
JMat JMat::clone(){
JMat cm(m_width,m_height,m_channel,m_stride,m_bit);
//printf("==clone %d\n",m_size);
memcpy(cm.m_buf,m_buf,m_size);
memcpy(cm.m_tagarr,m_tagarr,512*sizeof(int));
return cm;
}
#ifdef USE_OPENCV
JMat::JMat(std::string picfile,int flag):JBuf(){
cv::Mat image = cv::imread(picfile);
m_bit = flag?1:sizeof(float);
m_width = image.cols;
m_height = image.rows;
m_channel = 3;//image.channels();
//printf("===channels %d\n",m_channel);
m_stride = m_width*m_channel;
m_size = m_bit*m_stride*m_height;
m_buf = malloc(m_size+m_bit*m_stride);
m_ref = 0;
if(flag){
memcpy(m_buf,image.data,m_size);
//printf("===w %d h %d\n",image.cols,image.rows);
//cv::imshow("aaa",image);
//cv::waitKey(0);
//cv::Mat fmat(m_height,m_width,CV_8UC3,m_buf);
//float scale = 1.0f/255.0f;
//image.convertTo(fmat,CV_32F,scale);
}else{
cv::Mat fmat(m_height,m_width,CV_32FC3,m_buf);
float scale = 1.0f/255.0f;
image.convertTo(fmat,CV_32F,scale);
}
image.release();
init_tagarr();
}
#else
JMat::JMat(std::string picfile,int flag):JBuf(){
}
#endif
JMat::~JMat(){
}
float* JMat::fdata(){
return (float*)m_buf;
}
float* JMat::frow(int row){
return ((float*)m_buf)+ row*m_stride;
}
float* JMat::fitem(int row,int col){
return ((float*)m_buf)+ row*m_stride + col;
}
uint8_t* JMat::udata(){
return (uint8_t*)m_buf;
}
/*
nc::NdArray<float> JMat::ncarray(){
bool own = false;
nc::NdArray<float> arr = nc::NdArray<float>((float*)m_buf, m_height, m_width, own);
return arr;
}
*/
#ifdef USE_NCNN
ncnn::Mat JMat::packingmat(){
ncnn::Mat in_pack(m_width,m_height,1,(void*)m_buf,(size_t)4u*3,3);
ncnn::Mat in ;
ncnn::convert_packing(in_pack,in,1);
return in;
}
ncnn::Mat JMat::ncnnmat(){
unsigned char* data = (unsigned char*)m_buf;
if(m_channel == 3){
ncnn::Mat mat = ncnn::Mat::from_pixels(data, ncnn::Mat::PIXEL_BGR, m_width, m_height);
return mat;
}else if(m_channel == 4){
ncnn::Mat mat = ncnn::Mat::from_pixels(data, ncnn::Mat::PIXEL_BGRA, m_width, m_height);
return mat;
}else if(m_channel == 1){
ncnn::Mat mat = ncnn::Mat::from_pixels(data, ncnn::Mat::PIXEL_GRAY, m_width, m_height);
return mat;
}else {
ncnn::Mat mat = ncnn::Mat::from_pixels(data, ncnn::Mat::PIXEL_GRAY, m_width*m_channel, m_height);
return mat;
}
}
#endif

View File

@@ -0,0 +1,92 @@
#pragma once
#include <stdio.h>
#include <stdlib.h>
#include <stdio.h>
#include <memory>
#include <string>
#include <vector>
#include <string.h>
//#include "NumCpp.hpp"
#define USE_OPENCV
#define USE_NCNN
#define USE_TURBOJPG
#ifdef USE_OPENCV
#include "opencv2/core.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/highgui.hpp"
#endif
#ifdef USE_NCNN
#include "mat.h"
#endif
#ifdef USE_EIGEN
#include "eigen3/Eigen/Core"
typedef Eigen::Matrix<float, 1, Eigen::Dynamic, Eigen::RowMajor> Vectorf;
typedef Eigen::Matrix<std::complex<float>, 1, Eigen::Dynamic, Eigen::RowMajor> Vectorcf;
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor> Matrixf;
typedef Eigen::Matrix<std::complex<float>, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor> Matrixcf;
#endif
class JBuf{
protected:
bool m_ref = 0;
uint32_t m_size = 0;
void* m_buf = NULL;
public:
uint32_t size(){return m_size;} ;
void* data(){return m_buf;};
bool ref(){return m_ref;};
int zeros();
int forceref(int bref);
JBuf();
JBuf(uint32_t size,void* buf = nullptr);
virtual ~JBuf();
};
class JMat:public JBuf{
protected:
int m_bit = 0;
int m_width = 0;
int m_height = 0;
int m_channel = 0;
int m_stride = 0;
int m_tagarr[512];
void init_tagarr();
public:
int height(){return m_height;}
int width(){return m_width;}
int stride(){return m_stride;}
int channel(){return m_channel;}
JMat(int w,int h,float *buf ,int c = 3 ,int d = 0);
JMat(int w,int h,uint8_t *buf ,int c = 3 ,int d = 0);
JMat(int w,int h,int c = 3,int d = 0,int b=0);
JMat(std::string picfile,int flag=0);
JMat();
int load(std::string picfile);
int loadjpg(std::string picfile,int flag=0);
int savegpg(std::string gpgfile);
int loadgpg(std::string gpgfile);
float* fdata();
float* frow(int row);
float* fitem(int row,int col);
int tojpg(const char* fn);
int tobin(const char* fn);
int show(const char* title);
JMat clone();
JMat* refclone(int ref=1);
uint8_t* udata();
virtual ~JMat();
int* tagarr();
//nc::NdArray<float> ncarray();
#ifdef USE_OPENCV
cv::Mat cvmat();
#endif
#ifdef USE_NCNN
ncnn::Mat ncnnmat();
ncnn::Mat packingmat();
#endif
//Matrixf tomatrix();
};

View File

@@ -0,0 +1,229 @@
#include "malpha.h"
#include "blendgram.h"
#include "face_utils.h"
MWorkMat::MWorkMat(JMat* pic,JMat* msk,const int* boxs){
m_boxx = boxs[0];
m_boxy=boxs[1];
m_boxwidth=boxs[2]-m_boxx;
m_boxheight=boxs[3]-m_boxy;
//printf("x %d y %d w %d h %d \n",m_boxx,m_boxy,m_boxwidth,m_boxheight);
m_pic = pic;
m_msk = msk;
pic_real160 = new JMat(160,160,3,0,1);
pic_mask160 = new JMat(160,160,3,0,1);
//pic_crop160 = new JMat(160,160,3,0,1);
msk_real160 = new JMat(160,160,1,0,1);
//msk_mask160 = new JMat(160,160,3,0,1);
}
MWorkMat::~MWorkMat(){
matpic_org168.release();
matpic_roirst.release();
delete pic_real160;
delete pic_mask160;
delete msk_real160;
if(pic_clone160) delete pic_clone160;
}
int MWorkMat::munet(JMat** ppic,JMat** pmsk){
*ppic = pic_real160;
*pmsk = pic_mask160;
return 0;
}
int MWorkMat::premunet(){
matpic_roisrc = cv::Mat(m_pic->cvmat(),cv::Rect(m_boxx,m_boxy,m_boxwidth,m_boxheight));
cv::resize(matpic_roisrc , matpic_org168, cv::Size(168, 168), cv::INTER_AREA);
//vtacc
matpic_roi160 = cv::Mat(matpic_org168,cv::Rect(4,4,160,160));
cv::Mat cvmask = pic_mask160->cvmat();
cv::Mat cvreal = pic_real160->cvmat();
matpic_roi160.copyTo(cvmask);
matpic_roi160.copyTo(cvreal);
//cv::rectangle(cvmask,cv::Rect(5,5,150,150),cv::Scalar(0,0,0),-1);//,cv::LineTypes::FILLED);
cv::rectangle(cvmask,cv::Rect(5,5,150,145),cv::Scalar(0,0,0),-1);//,cv::LineTypes::FILLED);
//cv::rectangle(cvmask,cv::Rect(4,4,152,152),cv::Scalar(0,0,0),-1);//,cv::LineTypes::FILLED);
//cv::imwrite("cvmask.bmp",cvmask);
//cv::waitKey(0);
pic_clone160 = pic_real160->refclone(0);
return 0;
}
int MWorkMat::finmunet(JMat* fgpic){
cv::Mat cvreal = pic_real160->cvmat();
cvreal.copyTo(matpic_roi160);
//cv::imwrite("accpre.bmp",matpic_org168);
if(m_msk) vtacc((uint8_t*)matpic_org168.data,168*168);
//cv::imwrite("accend.bmp",matpic_org168);
cv::resize(matpic_org168, matpic_roirst, cv::Size(m_boxwidth, m_boxheight), cv::INTER_AREA);
if(fgpic){
matpic_roisrc = cv::Mat(fgpic->cvmat(),cv::Rect(m_boxx,m_boxy,m_boxwidth,m_boxheight));
matpic_roirst.copyTo(matpic_roisrc);
}else{
matpic_roirst.copyTo(matpic_roisrc);
}
return 0;
}
int MWorkMat::alpha(JMat** preal,JMat** pimg,JMat** pmsk){
*preal = pic_clone160;
*pimg = pic_real160;
*pmsk = msk_real160;
return 0;
}
int MWorkMat::prealpha(){
printf("x %d y %d w %d h %d \n",m_boxx,m_boxy,m_boxwidth,m_boxheight);
//m_msk->show("cba");
//cv::waitKey(0);
matmsk_roisrc = cv::Mat(m_msk->cvmat(),cv::Rect(m_boxx,m_boxy,m_boxwidth,m_boxheight));
cv::resize(matmsk_roisrc , matmsk_org168, cv::Size(168, 168), cv::INTER_AREA);
matmsk_roi160 = cv::Mat(matmsk_org168,cv::Rect(4,4,160,160));
cv::Mat cvmask = msk_real160->cvmat();
cv::cvtColor(matmsk_roi160,cvmask,cv::COLOR_RGB2GRAY);
//BlendGramAlphaRev(pic_clone160->udata(),msk_real160->udata(),pic_crop160->udata(),160,160);
//pic_crop160->show("aaa");
//cv::waitKey(0);
//pic_crop160
//
return 0;
}
int MWorkMat::finalpha(){
cv::Mat cvmask = msk_real160->cvmat();
cv::cvtColor(cvmask,matmsk_roi160,cv::COLOR_GRAY2RGB);
//
cv::resize(matmsk_org168, matmsk_roirst, cv::Size(m_boxwidth, m_boxheight), cv::INTER_AREA);
matmsk_roirst.copyTo(matmsk_roisrc);
return 0;
}
int MWorkMat::vtacc(uint8_t* buf,int count){
/*
int avgr = 0;
int avgb = 0;
int avgg = 0;
if(1){
uint8_t* pb = m_pic->udata();
for(int k=0;k<10;k++){
avgr += *pb++;
avgg += *pb++;
avgb += *pb++;
}
avgr =avgr/10 +10;
avgg =avgg/10 -20;
if(avgg<0)avgg=0;
avgb =avgb/10 + 10;
}
*/
uint8_t* pb = buf;
for(int k=0;k<count;k++){
int sum = (pb[0]+ pb[2])/2.0f;
if(pb[1]>=sum){
pb[1]=sum;
//pb[0]=0;
//pb[2]=0;
// }else if((pb[0]<avgr)&&(pb[1]>avgg)&&(pb[2]<avgb)){
//pb[1]=0;
//pb[0]=0;
//pb[2]=0;
}
pb+=3;
}
/*
long sum = 0l;
float mean = sum*0.5f/count;
uint8_t maxg = (mean>255.f)?255:mean;
//printf("sum %ld mean %f maxg %d\n",sum,mean,maxg);
//getchar();
pb = buf +1;
for(int k=0;k<count;k++){
if(*pb>maxg){
*pb = maxg;
}
pb+=3;
}
*/
return 0;
}
int MAlpha::doModel(JMat* real,JMat* img,JMat* pha){
if(1)return 0;
/*
if(1)return 0;
JMat picimg(160,160,3,0,1);
JMat picreal(160,160,3,0,1);
cv::cvtColor(real->cvmat(),picreal.cvmat(),cv::COLOR_RGB2BGR);
cv::cvtColor(img->cvmat(),picimg.cvmat(),cv::COLOR_RGB2BGR);
*/
float mean_vals[3] = {127.5f, 127.5f, 127.5f};
float norm_vals[3] = {1 / 127.5f, 1 / 127.5f, 1 / 127.5f};
ncnn::Mat inimg = ncnn::Mat::from_pixels(img->udata(), ncnn::Mat::PIXEL_BGR2RGB, 160, 160);
inimg.substract_mean_normalize(mean_vals, norm_vals);
ncnn::Mat inreal = ncnn::Mat::from_pixels(real->udata(), ncnn::Mat::PIXEL_BGR2RGB, 160, 160);
inreal.substract_mean_normalize(mean_vals, norm_vals);
ncnn::Mat inpha = ncnn::Mat::from_pixels(pha->udata(), ncnn::Mat::PIXEL_GRAY, 160, 160);
float gmean_vals[3] = {0.0f, 0.0f, 0.0f};
float gnorm_vals[3] = {1 / 255.0f, 1 / 255.0f, 1 / 255.0f};
inpha.substract_mean_normalize(gmean_vals, gnorm_vals);
ncnn::Mat inpic(160,160,7);
//printf("===in %d %d all %d %d pha %d %d\n",inreal.cstep,inreal.elempack,inpic.cstep,inpic.elempack,inpha.cstep,inpha.elempack);
//JMat picin(160,160,7);
float* buf = (float*)inpic.data;
memcpy(buf,inreal.data,inreal.cstep*inreal.c*sizeof(float));
buf += inpic.cstep*inreal.c;
memcpy(buf,inimg.data,inimg.cstep*inimg.c*sizeof(float));
buf += inimg.cstep*inimg.c;
memcpy(buf, inpha.data,inpha.cstep*sizeof(float));
//ncnn::Mat inpic(160,160,7,pd,4);
//ncnn::Mat inpack(160,160,1,pd,(size_t)4u*7,7);
//ncnn::Mat inpic;
//ncnn::convert_packing(inpack,inpic,1);
ncnn::Mat outpic;
ncnn::Extractor ex = net.create_extractor();
int rst = ex.input("input", inpic);
//printf("input %d\n",rst);
rst = ex.extract("output", outpic);
//printf("output %d\n",rst);
float outmean_vals[3] = {0.0f, 0.0f, 0.0f};
float outnorm_vals[3] = { 255.0f, 255.0f, 255.0f};
outpic.substract_mean_normalize(outmean_vals, outnorm_vals);
outpic.to_pixels(pha->udata(),ncnn::Mat::PIXEL_GRAY);
//pha->show("mmm");
//cv::waitKey(0);
//dumpfloat((float*)outpic.data+160*159,10);
//ncnn::Mat pakpic;
//ncnn::convert_packing(outpic,pakpic,1);
//dumpfloat((float*)pakpic.data,160*160*1);
//getchar();
/*
cv::Mat cvadj(160,160,CV_32FC1,outpic.data);
cv::Mat cvout;//(160,160,CV_8UC1);
float scale = 255.0f;//255.0f;
cvadj.convertTo(cvout,CV_8UC1,scale);
cvout.copyTo(pha->cvmat());
*/
//cv::imshow("pha",cvout);
//pha->show("pha");
//cv::waitKey(0);
return 0;
}
MAlpha::MAlpha(const char* fnbin,const char* fnparam):NcnnModel(160,160){
std::string fb(fnbin);
std::string fp(fnparam);
initModel(fb,fp);
}
MAlpha::~MAlpha(){
}

View File

@@ -0,0 +1,59 @@
#pragma once
#include "jmat.h"
#include "net.h"
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <stdio.h>
#include <vector>
#include "aimodel.h"
class MWorkMat{
private:
int m_boxx;
int m_boxy;
int m_boxwidth;
int m_boxheight;
JMat* m_pic;
JMat* m_msk;
JMat* pic_real160;//blendimg
JMat* pic_mask160;
cv::Mat matpic_roisrc;//box area
cv::Mat matpic_org168;
cv::Mat matpic_roi160;
JMat* pic_clone160;//blendimg
cv::Mat matpic_roirst;
//JMat* pic_crop160;
//
JMat* msk_real160;
//JMat* msk_mask160;
cv::Mat matmsk_roisrc;//box area
cv::Mat matmsk_org168;
cv::Mat matmsk_roi160;
cv::Mat matmsk_roirst;
int vtacc(uint8_t* buf,int count);
public:
MWorkMat(JMat* pic,JMat* msk,const int* boxs);
int premunet();
int munet(JMat** ppic,JMat** pmsk);
int finmunet(JMat* fgpic=NULL);
int prealpha();
int alpha(JMat** preal,JMat** pimg,JMat** pmsk);
int finalpha();
virtual ~MWorkMat();
};
class MAlpha:public NcnnModel{
private:
public:
int doModel(JMat* real,JMat* img,JMat* pha);
MAlpha(const char* fnbin,const char* fnparam);
virtual ~MAlpha();
};

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,375 @@
#pragma once
#ifndef SERVICESUPERVISOR_IIR_FILTER_H
#define SERVICESUPERVISOR_IIR_FILTER_H
//E(t,f) is computed using a first-order in-finite impulse response (IIR) filter
#define UES_IIR_I
//#define UES_IIR_II
#ifdef UES_IIR_I
class IIR_I
{
private:
double *m_pNum;
double *m_pDen;
double *m_px;
double *m_py;
int m_num_order;
int m_den_order;
public:
IIR_I();
~IIR_I();
void reset();
void setPara(double num[], int num_order, double den[], int den_order);
void resp(double data_in[], int m, double data_out[], int n);
void filter(double data_in[], double data_out[], int len);
};
/** \brief 将滤波器的内部状态清零,滤波器的系数保留
* \return
*/
void IIR_I::reset()
{
for(int i = 0; i <= m_num_order; i++)
{
m_pNum[i] = 0.0;
}
for(int i = 0; i <= m_den_order; i++)
{
m_pDen[i] = 0.0;
}
}
IIR_I::IIR_I()
{
m_pNum = NULL;
m_pDen = NULL;
m_px = NULL;
m_py = NULL;
m_num_order = -1;
m_den_order = -1;
};
IIR_I::~IIR_I()
{
delete[] m_pNum;
delete[] m_pDen;
delete[] m_px;
delete[] m_py;
m_pNum = NULL;
m_pDen = NULL;
m_px = NULL;
m_py = NULL;
};
/** \brief
*
* \param num 分子多项式的系数,升序排列,num[0] 为常数项
* \param m 分子多项式的阶数
* \param den 分母多项式的系数,升序排列,den[0] 为常数项
* \param m 分母多项式的阶数
* \return
*/
void IIR_I::setPara(double num[], int num_order, double den[], int den_order)
{
delete[] m_pNum;
delete[] m_pDen;
delete[] m_px;
delete[] m_py;
m_pNum = new double[num_order + 1];
m_pDen = new double[den_order + 1];
m_num_order = num_order;
m_den_order = den_order;
m_px = new double[num_order + 1];
m_py = new double[den_order + 1];
for(int i = 0; i < m_num_order; i++)
{
m_pNum[i] = num[i];
m_px[i] = 0.0;
}
m_pNum[m_num_order] = 0.0;
m_px[m_num_order] = 0.0;
for(int i = 0; i < m_den_order; i++)
{
m_pDen[i] = den[i];
m_py[i] = 0.0;
}
m_pDen[m_den_order] = 0.0;
m_py[m_den_order] = 0.0;
}
/** \brief 计算 IIR 滤波器的时域响应,不影响滤波器的内部状态
* \param data_in 为滤波器的输入0 时刻之前的输入默认为 0data_in[M] 及之后的输入默认为data_in[M-1]
* \param data_out 滤波器的输出
* \param M 输入数据的长度
* \param N 输出数据的长度
* \return
*/
void IIR_I::resp(double data_in[], int M, double data_out[], int N)
{
int i, k, il;
for(k = 0; k < N; k++)
{
data_out[k] = 0.0;
for(i = 0; i <= m_num_order; i++)
{
if( k - i >= 0)
{
il = ((k - i) < M) ? (k - i) : (M - 1);
data_out[k] = data_out[k] + m_pNum[i] * data_in[il];
}
}
for(i = 1; i <= m_den_order; i++)
{
if( k - i >= 0)
{
data_out[k] = data_out[k] - m_pDen[i] * data_out[k - i];
}
}
}
}
/** \brief 滤波函数采用直接I型结构
* 注该函数内部修改过移植librosa.pcen时参照scipy.signal.lfilter所做的设计。
*
* \param data_in[] 输入数据
* \param data_out[] 保存滤波后的数据
* \param len 数组的长度
* \return
*/
void IIR_I::filter(double data_in[], double data_out[], int len)
{
int i, k;
m_py[1] = 1; //修改的地方因为公式中y[n-k]当为第一个元素时会出现y[-1]pcen中y[-1]会被认为为1。
for(k = 0; k < len; k++)
{
m_px[0] = data_in[k];
m_py[0] = 0.0;
for(i = 0; i <= m_num_order; i++)
{
m_py[0] = m_py[0] + m_pNum[i] * m_px[i];
}
for(i = 1; i <= m_den_order; i++)
{
m_py[0] = m_py[0] - m_pDen[i] * m_py[i];
}
for(i = m_num_order; i >= 1; i--)
{
m_px[i] = m_px[i-1];
}
for(i = m_den_order; i >= 1; i--)
{
m_py[i] = m_py[i-1];
}
data_out[k] = m_py[0];
}
}
#endif
#ifdef UES_IIR_II
/**< IIR 滤波器直接II型实现 */
class IIR_II
{
public:
IIR_II();
void reset();
void setPara(double num[], int num_order, double den[], int den_order);
void resp(double data_in[], int m, double data_out[], int n);
double filter(double data);
void filter(double data[], int len);
void filter(double data_in[], double data_out[], int len);
protected:
private:
double *m_pNum;
double *m_pDen;
double *m_pW;
int m_num_order;
int m_den_order;
int m_N;
};
class IIR_BODE
{
private:
double *m_pNum;
double *m_pDen;
int m_num_order;
int m_den_order;
std::complex<double> poly_val(double p[], int order, double omega);
public:
IIR_BODE();
void setPara(double num[], int num_order, double den[], int den_order);
std::complex<double> bode(double omega);
void bode(double omega[], int n, std::complex<double> resp[]);
};
IIR_II::IIR_II()
{
//ctor
m_pNum = NULL;
m_pDen = NULL;
m_pW = NULL;
m_num_order = -1;
m_den_order = -1;
m_N = 0;
};
/** \brief 将滤波器的内部状态清零,滤波器的系数保留
* \return
*/
void IIR_II::reset()
{
for(int i = 0; i < m_N; i++)
{
m_pW[i] = 0.0;
}
}
/** \brief
*
* \param num 分子多项式的系数,升序排列,num[0] 为常数项
* \param m 分子多项式的阶数
* \param den 分母多项式的系数,升序排列,den[0] 为常数项
* \param m 分母多项式的阶数
* \return
*/
void IIR_II::setPara(double num[], int num_order, double den[], int den_order)
{
delete[] m_pNum;
delete[] m_pDen;
delete[] m_pW;
m_num_order = num_order;
m_den_order = den_order;
m_N = fmax(num_order, den_order) + 1;
m_pNum = new double[m_N];
m_pDen = new double[m_N];
m_pW = new double[m_N];
for(int i = 0; i < m_N; i++)
{
m_pNum[i] = 0.0;
m_pDen[i] = 0.0;
m_pW[i] = 0.0;
}
for(int i = 0; i <= num_order; i++)
{
m_pNum[i] = num[i];
}
for(int i = 0; i <= den_order; i++)
{
m_pDen[i] = den[i];
}
}
/** \brief 计算 IIR 滤波器的时域响应,不影响滤波器的内部状态
* \param data_in 为滤波器的输入0 时刻之前的输入默认为 0data_in[M] 及之后的输入默认为data_in[M-1]
* \param data_out 滤波器的输出
* \param M 输入数据的长度
* \param N 输出数据的长度
* \return
*/
void IIR_II::resp(double data_in[], int M, double data_out[], int N)
{
int i, k, il;
for(k = 0; k < N; k++)
{
data_out[k] = 0.0;
for(i = 0; i <= m_num_order; i++)
{
if( k - i >= 0)
{
il = ((k - i) < M) ? (k - i) : (M - 1);
data_out[k] = data_out[k] + m_pNum[i] * data_in[il];
}
}
for(i = 1; i <= m_den_order; i++)
{
if( k - i >= 0)
{
data_out[k] = data_out[k] - m_pDen[i] * data_out[k - i];
}
}
}
}
/** \brief 滤波函数采用直接II型结构
*
* \param data 输入数据
* \return 滤波后的结果
*/
double IIR_II::filter(double data)
{
m_pW[0] = data;
for(int i = 1; i <= m_den_order; i++) // 先更新 w[n] 的状态
{
m_pW[0] = m_pW[0] - m_pDen[i] * m_pW[i];
}
data = 0.0;
for(int i = 0; i <= m_num_order; i++)
{
data = data + m_pNum[i] * m_pW[i];
}
for(int i = m_N - 1; i >= 1; i--)
{
m_pW[i] = m_pW[i-1];
}
return data;
}
/** \brief 滤波函数采用直接II型结构
*
* \param data[] 传入输入数据,返回时给出滤波后的结果
* \param len data[] 数组的长度
* \return
*/
void IIR_II::filter(double data[], int len)
{
int i, k;
for(k = 0; k < len; k++)
{
m_pW[0] = data[k];
for(i = 1; i <= m_den_order; i++) // 先更新 w[n] 的状态
{
m_pW[0] = m_pW[0] - m_pDen[i] * m_pW[i];
}
data[k] = 0.0;
for(i = 0; i <= m_num_order; i++)
{
data[k] = data[k] + m_pNum[i] * m_pW[i];
}
for(i = m_N - 1; i >= 1; i--)
{
m_pW[i] = m_pW[i-1];
}
}
}
/** \brief 滤波函数采用直接II型结构
*
* \param data_in[] 输入数据
* \param data_out[] 保存滤波后的数据
* \param len 数组的长度
* \return
*/
void IIR_II::filter(double data_in[], double data_out[], int len)
{
int i, k;
for(k = 0; k < len; k++)
{
m_pW[0] = data_in[k];
for(i = 1; i <= m_den_order; i++) // 先更新 w[n] 的状态
{
m_pW[0] = m_pW[0] - m_pDen[i] * m_pW[i];
}
data_out[k] = 0.0;
for(i = 0; i <= m_num_order; i++)
{
data_out[k] = data_out[k] + m_pNum[i] * m_pW[i];
}
for(i = m_N - 1; i >= 1; i--)
{
m_pW[i] = m_pW[i-1];
}
}
}
#endif
#endif //SERVICESUPERVISOR_IIR_FILTER_H

View File

@@ -0,0 +1,441 @@
#pragma once
#include "AudioFFT.hpp"
//#include"../third/numcpp/NumCpp.hpp"
#include "opencv2/opencv.hpp"
#include "iir_filter.hpp"
#include "sas_util.h"
int nSamplesPerSec = 16000; //采样率(每秒样本数) //Sample rate.(keda, thchs30, aishell)
int length_DFT = 1024;//2048; //傅里叶点数 //fft points (samples)
int hop_length = 160;//int(0.05 * nSamplesPerSec); //步长 //下一帧取数据相对于这一帧的右偏移量
int win_length = 800;// int(0.1 * nSamplesPerSec); //帧长 //假设16000采样率则取取0.1s时间的数据
int number_filterbanks = 80; //过滤器数量 //Number of Mel banks to generate
float preemphasis = 0.97; //预加重(高通滤波器比例值)
int max_db = 100;
int ref_db = 20;
int r = 1; //librosa里的r=1暂未深入分析其作用
double pi = 3.14159265358979323846;
cv::Mat_<double> mel_basis;
cv::Mat_<float> hannWindow;
std::shared_ptr<IIR_I> filter;
//"""Convert Hz to Mels"""
double hz_to_mel(double frequencies, bool htk = false) {
if (htk) {
return 2595.0 * log10(1.0 + frequencies / 700.0);
}
// Fill in the linear part
double f_min = 0.0;
double f_sp = 200.0 / 3;
double mels = (frequencies - f_min) / f_sp;
// Fill in the log-scale part
double min_log_hz = 1000.0; // beginning of log region (Hz)
double min_log_mel = (min_log_hz - f_min) / f_sp; // same (Mels)
double logstep = log(6.4) / 27.0; // step size for log region
// 对照Python平台的librosa库移植
//如果是多维数列
// if (frequencies.ndim) {
// // If we have array data, vectorize
// log_t = (frequencies >= min_log_hz)
// mels[log_t] = min_log_mel + np.log(frequencies[log_t] / min_log_hz) / logstep
// } else
if (frequencies >= min_log_hz) {
// If we have scalar data, heck directly
mels = min_log_mel + log(frequencies / min_log_hz) / logstep;
}
return mels;
}
//"""Convert mel bin numbers to frequencies"""
cv::Mat_<double> mel_to_hz(cv::Mat_<double> mels, bool htk = false) {
// if (htk) {
// return //python://700.0 * (10.0**(mels / 2595.0) - 1.0);
// }
// Fill in the linear scale
double f_min = 0.0;
double f_sp = 200.0 / 3;
cv::Mat_<double> freqs = mels * f_sp + f_min;
// And now the nonlinear scale
double min_log_hz = 1000.0; // beginning of log region (Hz)
double min_log_mel = (min_log_hz - f_min) / f_sp; // same (Mels)
double logstep = log(6.4) / 27.0; // step size for log region
// 对照Python平台的librosa库移植
//if (mels.ndim) {
// If we have vector data, vectorize
cv::Mat_<bool> log_t = (mels >= min_log_mel);
for (int i = 0; i < log_t.cols; i++) {
if (log_t(0, i)) {
freqs(0, i) = cv::exp((mels(0, i) - min_log_mel) * logstep) * min_log_hz;
}
}
//}
return freqs;
}
// 生成等差数列类似np.linspace
cv::Mat_<double> cvlinspace(double min_, double max_, int length) {
auto cvmat = cv::Mat_<double>(1, length);
for (int i = 0; i < length; i++) {
cvmat(0, i) = ((max_ - min_) / (length - 1) * i) + min_;
}
return cvmat;
}
//"""Create a Filterbank matrix to combine FFT bins into Mel-frequency bins"""
cv::Mat_<double> mel_spectrogram_create(int nps, int n_fft, int n_mels) {
double f_max = nps / 2.0;
double f_min = 0;
int n_fft_2 = 1 + n_fft / 2;
// Initialize the weights
//auto weights = nc::zeros<double>(nc::uint32(n_mels), nc::uint32(n_fft_2));
auto weights = cv::Mat_<double>(n_mels, n_fft_2, 0.0);
// Center freqs of each FFT bin
//auto fftfreqs_ = nc::linspace<double>(f_min, f_max, nc::uint32(n_fft_2), true);
auto fftfreqs = cvlinspace(f_min, f_max, n_fft_2);
// 'Center freqs' of mel bands - uniformly spaced between limits
double min_mel = hz_to_mel(f_min, false);
double max_mel = hz_to_mel(f_max, false);
//auto mels_ = nc::linspace(min_mel, max_mel, nc::uint32(n_mels + 2));
auto mels = cvlinspace(min_mel, max_mel, n_mels + 2);
auto mel_f = mel_to_hz(mels, false);
//auto fdiff_ = nc::diff(mel_f_); //沿着指定轴计算第N维的离散差值(后一个元素减去前一个元素)
cv::Mat_<double> d1(1, mel_f.cols * mel_f.rows - 1, (double *) (mel_f.data) + 1);
cv::Mat_<double> d2(1, mel_f.cols * mel_f.rows - 1, (double *) (mel_f.data));
cv::Mat_<double> fdiff = d1 - d2;
//auto ramps = nc::subtract.outer(mel_f, fftfreqs); //nc没有subtract.outer
//nc::NdArray<double> ramps = nc::zeros<double>(mel_f.cols, fftfreqs.cols);
auto ramps = cv::Mat_<double>(mel_f.cols, fftfreqs.cols);
for (int i = 0; i < mel_f.cols; i++) {
for (int j = 0; j < fftfreqs.cols; j++) {
ramps(i, j) = mel_f(0, i) - fftfreqs(0, j);
}
}
for (int i = 0; i < n_mels; i++) {
// lower and upper slopes for all bins
//auto ramps_1 = nc::NdArray<double>(1, ramps.cols);
auto ramps_1 = cv::Mat_<double>(1, ramps.cols);
for (int j = 0; j < ramps.cols; j++) {
ramps_1(0, j) = ramps(i, j);
}
//auto ramps_2 = nc::NdArray<double>(1, ramps.cols);
auto ramps_2 = cv::Mat_<double>(1, ramps.cols);
for (int j = 0; j < ramps.cols; j++) {
ramps_2(0, j) = ramps(i + 2, j);
}
cv::Mat_<double> lower = ramps_1 * -1 / fdiff(0, i);
cv::Mat_<double> upper = ramps_2 / fdiff(0, i + 1);
// .. then intersect them with each other and zero
//auto weights_1 = nc::maximum(nc::zeros<double>(1, ramps.cols), nc::minimum(lower, upper));
cv::Mat c1 = lower;//(cv::Mat_<double>(1,5) << 1,2,-3,4,-5);
cv::Mat c2 = upper;
cv::Mat weights_1 = cv::Mat_<double>(1, lower.cols);
cv::min(c1, c2, weights_1);
cv::max(weights_1, 0, weights_1);
for (int j = 0; j < n_fft_2; j++) {
weights(i, j) = weights_1.at<double_t>(0, j);
}
}
// Slaney-style mel is scaled to be approx constant energy per channel
auto enorm = cv::Mat_<double>(1, n_mels);
for (int j = 0; j < n_mels; j++) {
enorm(0, j) = 2.0 / (mel_f(0, j + 2) - mel_f(0, j));
}
for (int j = 0; j < n_mels; j++) {
for (int k = 0; k < n_fft_2; k++) {
weights(j, k) *= enorm(0, j);
}
}
return weights;
}
//"""Short-time Fourier transform (STFT)""": 默认center=True, window='hann', pad_mode='reflect'
cv::Mat_<double> MagnitudeSpectrogram(const cv::Mat_<float> *emphasis_data, int n_fft = 2048, int hop_length = 0,
int win_length = 0) {
if (win_length == 0) {
win_length = n_fft;
}
if (hop_length == 0) {
hop_length = win_length / 4;
}
// reflect对称填充
int pad_lenght = n_fft / 2;
// 使用opencv里的copyMakeBorder来完成reflect填充
cv::Mat_<float> cv_padbuffer;
cv::copyMakeBorder(*emphasis_data, cv_padbuffer, 0, 0, pad_lenght, pad_lenght, cv::BORDER_REFLECT_101);
// windowing加窗将每一帧乘以汉宁窗以增加帧左端和右端的连续性。
// 生成一个1600长度的hannWindow并居中到2048长度的
if (hannWindow.empty()) {
hannWindow = cv::Mat_<float>(1, n_fft, 0.0f);
int insert_cnt = 0;
if (n_fft > win_length) {
insert_cnt = (n_fft - win_length) / 2;
} else {
//std::cout << "\tn_fft:" << n_fft << " > win_length:" << n_fft << std::endl;
return cv::Mat_<double>(0, 0);
}
for (int k = 1; k <= win_length; k++) {
hannWindow(0, k - 1 + insert_cnt) = float(0.5 * (1 - cos(2 * pi * k / (win_length + 1))));
}
}
// opencv虽然有Hann窗生成函数但是必须要求width > 1height > 1
//cv::Mat_<double> cv_hannWindow;
//cv::createHanningWindow(cv_hannWindow, cv::Size(1, win_length), CV_64FC1);
int size = cv_padbuffer.rows * cv_padbuffer.cols;//padbuffer.size()
int number_feature_vectors = (size - n_fft) / hop_length + 1;
int number_coefficients = n_fft / 2 + 1;
cv::Mat_<float> feature_vector(number_feature_vectors, number_coefficients, 0.0f);
audiofft::AudioFFT fft; //将FFT初始化放在循环外可达到最优速度
fft.init(size_t(n_fft));
for (int i = 0; i <= size - n_fft; i += hop_length) {
// 每次取一段数据
cv::Mat_<float> framef = cv::Mat_<float>(1, n_fft, (float *) (cv_padbuffer.data) + i).clone();
// 加hann窗
framef = framef.mul(hannWindow);
// 复数Xrf实数Xif虚数。
cv::Mat_<float> Xrf(1, number_coefficients);
cv::Mat_<float> Xif(1, number_coefficients);
fft.fft((float *) (framef.data), (float *) (Xrf.data), (float *) (Xif.data));
// 求模
cv::pow(Xrf, 2, Xrf);
cv::pow(Xif, 2, Xif);
cv::Mat_<float> cv_feature(1, number_coefficients, &(feature_vector[i / hop_length][0]));
cv::sqrt(Xrf + Xif, cv_feature);
}
cv::Mat_<float> cv_mag;
cv::transpose(feature_vector, cv_mag);
cv::Mat_<double> mag;
cv_mag.convertTo(mag, CV_64FC1);
return mag;
}
/*********************************************
* 名称log_mel
* 功能传入音频数据输出log-mel方式提取的特征数据。
* 参数:@ifile_data 传入的音频数据
* @nSamples_per_sec 音频采样率
* 返回cv::Mat_<double> 特征数据
*********************************************/
//cv::Mat_<double> log_mel(std::vector<uint8_t> &ifile_data, int nSamples_per_sec) {
int log_mel(float* ifile_data, int ifile_length,int nSamples_per_sec,float* ofile_data) {
if (nSamples_per_sec != nSamplesPerSec) {
//std::cout << R"(the "nSamples_per_sec" is not 16000.)" << std::endl;
return -1;//cv::Mat_<double>(0, 0);
}
//int ifile_length = int(ifile_data.size() / 4);
// pre-emphasis 预加重 //高通滤波
//cv::Mat_<float> d1(1, ifile_length - 1, (float *) (ifile_data.data()) + 1);
//cv::Mat_<float> d2(1, ifile_length-1 , (float *) (ifile_data.data()));
cv::Mat_<float> d1(1, ifile_length - 1, (float *) (ifile_data) + 1);
cv::Mat_<float> d2(1, ifile_length-1 , (float *) (ifile_data));
//std::cout<<ifile_length<<"====="<<d1[0][960000-1]<<std::endl;
cv::Mat_<float> cv_emphasis_data;
cv::hconcat(cv::Mat_<float>::zeros(1, 1), d1 - d2 * preemphasis, cv_emphasis_data);
//cv::print(cv_emphasis_data);
//std::cout<<ifile_length<<"====="<< cv_emphasis_data[0][960000-1]<<std::endl;
// magnitude spectrogram 幅度谱图
auto mag = MagnitudeSpectrogram(&cv_emphasis_data, length_DFT, hop_length, win_length);
auto magb = cv::abs(mag);
cv::pow(magb,2,mag);
//tooken
// 生成梅尔谱图 mel spectrogram //3ms
if (mel_basis.empty()) {
mel_basis = mel_spectrogram_create(nSamplesPerSec, length_DFT, number_filterbanks);
}
//cv::print(mel_basis);
//std::cout<<mel_basis.cols<<"=====cv_mel"<<mel_basis.rows<<std::endl;
// doc
cv::Mat cv_mel = mel_basis * mag;
//cv::Mat cv_mel = mel_basis.dot( mag);
// to decibel
//mel = 20 * np.log10(np.maximum(1e-5, mel))
//mag = 20 * np.log10(np.maximum(1e-5, mag))
//由于后续没用用到mag了所以不再对mag做运算。
// 使用opencv来实现
//cv::log(cv::max(cv_mel, 1e-5), cv_mel);
//cv::log(cv::max(cv_mel, 1e-5), cv_mel);
cv::log(cv_mel+ 1e-5, cv_mel);
// opencv没有log10()所以使用log(x)/log(10)来运算。
cv_mel = cv_mel / 2.3025850929940459 * 10; // 2.3025850929940459=log(10)
// normalize
//mel = np.clip((mel - hp.ref_db + hp.max_db) / hp.max_db, 1e-8, 1)
//mag = np.clip((mag - hp.ref_db + hp.max_db) / hp.max_db, 1e-8, 1)
//cv::normalize(cv_mel, cv_mel, 1e-8, 1.0, cv::NORM_MINMAX); // cv::normalize无法实现
//cv_mel = (cv_mel - ref_db + max_db) / max_db;
//cv_mel = cv::max(cv::min(cv_mel, 1.0), 1e-8);
cv_mel = cv_mel - ref_db;
//cv::print(cv_mel);
//std::cout<<cv_mel.cols<<"=====cv_mel"<<cv_mel.rows<<std::endl;
//std::cout<<"=====cv_mel"<<std::endl;
//getchar();
// Transpose
//mel = mel.T.astype(np.float32)
//mag = mag.T.astype(np.float32)
// 使用opencv的transpose
cv::Mat cv_mel_r;//(cv_mel.cols,cv_mel.rows,CV_64FC1,ofile_data);
cv::transpose(cv_mel, cv_mel_r);
//cv::Mat rcv(cv_mel_r.cols,cv_mel_r.rows, CV_32FC1,ofile_data);
cv::Mat rrr(cv_mel.cols,cv_mel.rows,CV_32FC1,ofile_data);
cv_mel_r.convertTo(rrr, CV_32FC1);
if (r == 1) {
// 原计算公式是:
// mel = mel[:len(mel) // hp.r * hp.r].reshape([len(mel) // hp.r, hp.r * hp.n_mels])
// 当r=1的时候公式运算无任何数值改变。
} else {
//std::cout << R"(the "r" is not 1.)" << std::endl;
}
// 返回mel特征向量
return 0;
}
/**--------------------------------- 以下是pcen运算方法 ---------------------------------**/
// scipy.signal.lfilter_zi()
cv::Mat_<double> cvlfilter_zi(cv::Mat_<double> b, cv::Mat_<double> a) {
if ((b.rows != 1) || (a.rows != 1)) {
//std::cout << "Numerator b and Denominator a must be 1-D." << std::endl;
}
if (a(0, 0) != 1) {
// Normalize the coefficients so a[0] == 1.
b = b / a(0, 0);
a = a / a(0, 0);
}
int len_a = a.cols * a.rows;
int len_b = b.cols * b.rows;
int n = len_a > len_b ? len_a : len_b;
if (len_a < n) {
cv::hconcat(a, cv::Mat_<float>::zeros(1, n - len_a), a);
} else if (len_b < n) {
cv::hconcat(b, cv::Mat_<float>::zeros(1, n - len_b), b);
}
return cv::Mat_<double>(0, 0);
}
/*
// scipy.signal.lfilter()
// Filter data along one-dimension with an IIR or FIR filter.
cv::Mat_<double> cvlfilter(cv::Mat_<double> &b, cv::Mat_<double> &a, cv::Mat_<double> &x,
cv::Mat_<double> &zi, int axis = -1) {
if (a.rows * a.cols == 1) {
// This path only supports types fdgFDGO to mirror _linear_filter below.
// Any of b, a, x, or zi can set the dtype, but there is no default
// casting of other types; instead a NotImplementedError is raised.
// 后续如果需要,则进行补充
} else {
// return sigtools._linear_filter(b, a, x, axis, zi)
// sigtools._linear_filter()
// (y,Vf) = _linear_filter(b,a,X,Dim=-1,Vi=None) implemented using Direct Form II transposed flow diagram.
// If Vi is not given, Vf is not returned.
;
}
}
*/
/*********************************************
* 名称pcen
* 功能传入音频数据输出pcen方式提取的特征数据。
* 参数:@ifile_data 传入的音频数据
* @nSamples_per_sec 音频采样率
* 返回cv::Mat_<double> 特征数据
*********************************************/
cv::Mat_<double> pcen(std::vector<uint8_t> &ifile_data, int nSamples_per_sec) {
//if (!(&ifile_data) || ifile_data.empty()) {
if (ifile_data.empty()) {
//std::cout << "error: invalid paramter: ifile_data" << std::endl;
return cv::Mat_<double>(0, 0);
}
if (nSamples_per_sec != nSamplesPerSec) {
// std::cout << R"(error: the "nSamples_per_sec" is not 16000.)" << std::endl;
return cv::Mat_<double>(0, 0);
}
int ifile_length = int(ifile_data.size() / 4);
cv::Mat_<float> cv_emphasis_data(1, ifile_length, (float *) (ifile_data.data()));
// std::cout<<ifile_length<<"====="<<cv_emphasis_data[0][960000-1]<<std::endl;
//getchar();
// magnitude spectrogram 幅度谱图
auto mag = MagnitudeSpectrogram(&cv_emphasis_data, length_DFT, hop_length, win_length);
mag = cv::abs(mag) * std::pow(2, 31);
// 生成梅尔谱图 mel spectrogram //3ms
if (mel_basis.empty()) {
mel_basis = mel_spectrogram_create(nSamplesPerSec, length_DFT, number_filterbanks);
}
// doc
cv::Mat_<double> mel = mel_basis * mag;
// 计算pcen特征
// double time_constant = 0.400;
// int sr = 22050;
// int hop_length = 512;
// double t_frames = time_constant * sr / double(hop_length);
// double b = (sqrt(1 + 4 * t_frames * t_frames) - 1) / (2 * t_frames * t_frames);
// cv::Mat_<double> zi = (cv::Mat_<double>(1, 1) << 0.94361056);
//
// cv::Mat_<double> in_b = (cv::Mat_<double>(1, 1) << b);
// cv::Mat_<double> in_a = (cv::Mat_<double>(1, 2) << 1, b - 1);
// cv::Mat_<double> zi = cvlfilter_zi(in_b, in_a);
// 第二个公式计算
// cv::Mat_<double> S_smooth = cvlfilter(in_b, in_a, mel, zi);
#if 1 // IIR滤波器
if (!filter) {
filter = std::make_shared<IIR_I>();
double iir_b[1] = {0.05638943879134889};
double iir_a[2] = {1.0, -0.9436105612086512};
//filter.reset();
filter->setPara(iir_b, 1, iir_a, 2);
}
cv::Mat_<double> S_smooth = cv::Mat_<double>(mel.rows, mel.cols);
for (int i = 0; i < mel.rows; i++) {
filter->filter(mel[i], S_smooth[i], mel.cols);
}
#endif
// 第一个公式计算
double gain = 0.98;
double bias = 2.0;
double power = 0.5;
double eps = 1e-6;
//python: smooth = np.exp(-gain * (np.log(eps) + np.log1p(S_smooth / eps)))
cv::Mat_<double> S_smooth_log1p;
cv::log(S_smooth / eps + 1, S_smooth_log1p);
cv::Mat_<double> smooth;
cv::exp((S_smooth_log1p + cv::log(eps)) * (-gain), smooth);
//python: S_out = (bias ** power) * np.expm1(power * np.log1p(ref * smooth / bias))
cv::Mat_<double> smooth_log1p;
cv::Mat_<double> smooth_log1p_exp;
cv::log(mel.mul(smooth) / bias + 1, smooth_log1p);
cv::exp(power * smooth_log1p, smooth_log1p_exp);
cv::Mat_<double> S_out = (smooth_log1p_exp - 1) * pow(bias, power);
// transpose
cv::Mat_<double> pcen;
cv::transpose(S_out, pcen);
return pcen;
}

View File

@@ -0,0 +1,123 @@
#pragma once
#include <string>
#include <chrono>
#include <vector>
#include <assert.h>
#include <memory>
#include <fstream>
#include "opencv2/opencv.hpp"
using namespace std;
using namespace std::chrono;
class parambase
{
public:
string name;
string help;
string strval;
parambase(){}
virtual ~parambase(){}
virtual bool set(const char* value) {return true;};
};
/**
* 引擎参数
*/
class EnginePar
{
public:
static int cs_timeout; //叫号服务完成的超时时间(默认情况下下一次叫号代表上一次完成,默认值5分钟)
static int cs_detecthandsup_time; //叫号后持续检测举手的时间(默认10s)
static int cs_detecthandsup_interval ; //叫号后持续检测举手的时间间隔(默认1秒1次)
static int cs_detectsmile_interval; //叫号后微笑检测的时间间隔(默认1秒1次)
static int cs_detectspeech_interval;//叫号后语音检测的时间间隔(默认20秒)
static int cs_detectpose_interval; //叫号后姿态检测的时间间隔(默认5秒1次)
static int detectpose_interval; //非叫号期间姿态检测的时间间隔(默认5秒1次)
static int detectsmile_interval; //非叫号期间微笑检测的时间间隔(默认1秒1次)
static int detectappearance_interval; //着装检测间隔
static float action_turnpen_thrd; //转笔阈值
static float action_turnchair_thrd; //转椅阈值
static float action_record_time; //动作录制时长
static float sit_supporthead_thrd; //撑头阈值
static float sit_layondesk_thrd; //趴桌阈值
static float sit_relyingonchair_thrd;//靠椅阈值
static string log_path;
static string log_level;
static string temp_path;
static bool set(const char* key, const char* val);
static bool haskey(const char* key);
static const char* getvalue(const char* key);
};
/**
* 摄像头分析场景
*/
enum VideoScene
{
SCENE_counter, // 柜台
SCENE_financial, // 理财
SCENE_lobby, // 大堂
SCENE_hall // 门厅
};
/**
* 摄像头分析参数
*/
class VideoPar
{
private:
vector<shared_ptr<parambase>> params;
public:
VideoScene scene; //场景: 1柜台, 2理财, 3大堂, 4进门(着装检测)
bool audio_enable ; //音频开关 1开,0关
int audio_channels ; //音频通道数 0,1,2,4,6
int audio_sample_rate ; // 采样率 44100, 48000, 96000, 192000
bool video_enable ; // 视频开关 1开,0关
//int video_analyse_rate ; //视频分析速率: 数字>0,每秒分析帧数
bool video_sample_keyframe; //只解码关键帧
bool video_record; //启用录制视频 1开,0关
int video_record_duration; //视频录制时长,默认10s
int video_record_reviewtime; //视频录制回溯时长,默认5s
int face_minsize; //最小人脸大小
VideoPar();
//~VideoPar();
bool set(const char* key, const char* val);
static bool haskey(const char* key);
};
template<class T>
inline int64_t NowTime()
{
return std::chrono::time_point_cast<T>(std::chrono::system_clock::now()).time_since_epoch().count();
}
/**--------------------------------- 以下是models各个模型所用到的方法 ---------------------------------**/
inline bool detectFileExist(char *file_path) {
std::ifstream _ifstream;
_ifstream.open(file_path, std::ios::in);
if (!_ifstream) {
return false;
}
_ifstream.close();
return true;
}
// 矩阵变换对向量xy进行旋转
inline cv::Mat_<double> rotate_point(cv::Mat_<double> xy, double angle) {
cv::Mat rotate_matrix = (cv::Mat_<double>(2, 2) << cos(angle), -sin(angle), sin(angle), cos(angle));
cv::transpose(rotate_matrix, rotate_matrix);
auto rotate_xy = xy * rotate_matrix;
return rotate_xy;
}
// 检查点是否在框内
inline bool check_point_in_rect(cv::Point point, cv::Rect rect) {
if ((rect.x < point.x && point.x < rect.x + rect.width) &&
(rect.y < point.y && point.y < rect.y + rect.height)) {
return true;//在rect内部
} else {
return false;//在rect边上或外部
}
}

View File

@@ -0,0 +1,332 @@
#include "munet.h"
#include "cpu.h"
#include "face_utils.h"
#include "blendgram.h"
Mobunet::Mobunet(const char* fnbin,const char* fnparam,const char* fnmsk){
initModel(fnbin,fnparam,fnmsk);
}
Mobunet::Mobunet(const char* modeldir,const char* modelid){
char fnbin[1024];
char fnparam[1024];
char fnmsk[1024];
sprintf(fnbin,"%s/%s.bin",modeldir,modelid);
sprintf(fnparam,"%s/%s.param",modeldir,modelid);
sprintf(fnmsk,"%s/weight_168u.bin",modeldir);
initModel(fnbin,fnparam,fnmsk);
}
int Mobunet::initModel(const char* binfn,const char* paramfn,const char* mskfn){
unet.clear();
//ncnn::set_cpu_powersave(2);
//ncnn::set_omp_num_threads(ncnn::get_big_cpu_count());
//unet.opt = ncnn::Option();
//unet.opt.use_vulkan_compute = true;
unet.opt.num_threads = ncnn::get_big_cpu_count();
//unet.load_param("model/mobileunet_v5_wenet_sim.param");
//unet.load_model("model/mobileunet_v5_wenet_sim.bin");
unet.load_param(paramfn);
unet.load_model(binfn);
char* wbuf = NULL;
dumpfile((char*)mskfn,&wbuf);
mat_weights = new JMat(160,160,(uint8_t*)wbuf,1);
mat_weights->forceref(0);
//mat_weights->show("weight");
//cv::waitKey(0);
return 0;
}
Mobunet::~Mobunet(){
unet.clear();
if(mat_weights){
delete mat_weights;
mat_weights = nullptr;
}
}
int Mobunet::domodelold(JMat* pic,JMat* msk,JMat* feat){
JMat picall(160*160,2,3,0,1);
uint8_t* buf = picall.udata();
int width = pic->width();
int height = pic->height();
cv::Mat c1(height,width,CV_8UC3,buf);
cv::Mat c2(height,width,CV_8UC3,buf+width*height*3);
cv::cvtColor(pic->cvmat(),c1,cv::COLOR_RGB2BGR);
cv::cvtColor(msk->cvmat(),c2,cv::COLOR_RGB2BGR);
ncnn::Mat inall = ncnn::Mat::from_pixels(buf, ncnn::Mat::PIXEL_BGR, 160*160, 2);
inall.substract_mean_normalize(mean_vals, norm_vals);
//inall.reshape(160,160,6);
ncnn::Mat inwenet(256,20,1,feat->data());
ncnn::Mat outpic;
ncnn::Extractor ex = unet.create_extractor();
ex.input("face", inall);
ex.input("audio", inwenet);
ex.extract("output", outpic);
float outmean_vals[3] = {-1.0f, -1.0f, -1.0f};
float outnorm_vals[3] = { 0.5f, 0.5f, 0.5f};
outpic.substract_mean_normalize(outmean_vals, outnorm_vals);
ncnn::Mat pakpic;
ncnn::convert_packing(outpic,pakpic,3);
cv::Mat cvadj(160,160,CV_32FC3,pakpic.data);
//dumpfloat((float*)cvadj.data,160*160*3);
cv::Mat cvreal;
float scale = 255.0f;
cvadj.convertTo(cvreal,CV_8UC3,scale);
cv::Mat cvmask;
cv::cvtColor(cvreal,cvmask,cv::COLOR_RGB2BGR);
cv::imshow("cvreal",cvreal);
cv::imshow("cvmask",cvmask);
//cv::waitKey(0);
//getchar();
BlendGramAlpha((uchar*)cvmask.data,(uchar*)mat_weights->data(),(uchar*)pic->data(),160,160);
return 0;
}
int Mobunet::domodel(JMat* pic,JMat* msk,JMat* feat){
//convert to bgr
//pic->tojpg("eee.bmp");
//JMat picmask(160,160,3,0,1);
//JMat picreal(160,160,3,0,1);
//cv::cvtColor(pic->cvmat(),picreal.cvmat(),cv::COLOR_RGB2BGR);
//cv::cvtColor(msk->cvmat(),picmask.cvmat(),cv::COLOR_RGB2BGR);
ncnn::Mat inmask = ncnn::Mat::from_pixels(msk->udata(), ncnn::Mat::PIXEL_BGR2RGB, 160, 160);
inmask.substract_mean_normalize(mean_vals, norm_vals);
ncnn::Mat inreal = ncnn::Mat::from_pixels(pic->udata(), ncnn::Mat::PIXEL_BGR2RGB, 160, 160);
inreal.substract_mean_normalize(mean_vals, norm_vals);
//
//JMat picin(160*160,6);
//char* pd = (char*)picin.data();
//memcpy(pd,inreal.data,160*160*3*4);
//memcpy(pd+ 160*160*3*4,inmask.data,160*160*3*4);
//
//ncnn::Mat inpack(160,160,1,pd,(size_t)4u*6,6);
//ncnn::Mat inpic;
//ncnn::convert_packing(inpack,inpic,1);
ncnn::Mat inpic(160,160,6);
//printf("===in %d %d all %d %d\n",inreal.cstep,inreal.elempack,inpic.cstep,inpic.elempack);
float* buf = (float*)inpic.data;
float* pr = (float*)inreal.data;
//printf("==%d==%f\n",pic->udata()[2],*pr);
memcpy(buf,pr,inreal.cstep*sizeof(float)*inreal.c);
/*
for(int k=0;k<3;k++){
memcpy(buf,pr,inreal.cstep*sizeof(float));
buf += inpic.cstep;
pr += inreal.cstep;
}
*/
buf+= inpic.cstep*inreal.c;
float* pm = (float*)inmask.data;
//printf("=%d===%f\n",msk->udata()[2],*pm);
memcpy(buf,pm,inmask.cstep*sizeof(float)*inmask.c);
/*
for(int k=0;k<3;k++){
memcpy(buf,pm,inreal.cstep*sizeof(float));
buf += inpic.cstep;
pm += inmask.cstep;
}
*/
ncnn::Mat inwenet(256,20,1,feat->data());
//ncnn::Mat inwenet(20,256,1,feat->data());
ncnn::Mat outpic;
ncnn::Extractor ex = unet.create_extractor();
ex.input("face", inpic);
ex.input("audio", inwenet);
ex.extract("output", outpic);
float outmean_vals[3] = {-1.0f, -1.0f, -1.0f};
float outnorm_vals[3] = { 127.5f, 127.5f, 127.5f};
outpic.substract_mean_normalize(outmean_vals, outnorm_vals);
cv::Mat cvout(160,160,CV_8UC3);
outpic.to_pixels(cvout.data,ncnn::Mat::PIXEL_RGB2BGR);
BlendGramAlpha((uchar*)cvout.data,(uchar*)mat_weights->data(),(uchar*)pic->data(),160,160);
//pic->tojpg("fff.bmp");
//getchar();
/*
ncnn::Mat pakpic;
ncnn::convert_packing(outpic,pakpic,3);
cv::Mat cvadj(160,160,CV_32FC3,pakpic.data);
//dumpfloat((float*)cvadj.data,160*160*3);
float scale = 255.0f;
cvadj.convertTo(picreal.cvmat(),CV_8UC3,scale);
cv::cvtColor(picreal.cvmat(),picmask.cvmat(),cv::COLOR_RGB2BGR);
*/
//getchar();
//getchar();
//cv::Mat cout;
//cv::cvtColor(picreal.cvmat(),cout,cv::COLOR_RGB2BGR);
//BlendGramAlpha((uchar*)cout.data,(uchar*)mat_weights->data(),(uchar*)pic->data(),160,160);
/*
float outmean_vals[3] = {-1.0f, -1.0f, -1.0f};
// float outnorm_vals[3] = { 2.0f, 2.0f, 2.0f};
float outnorm_vals[3] = { 127.5f, 127.5f, 127.5f};
outpic.substract_mean_normalize(outmean_vals, outnorm_vals);
ncnn::Mat pakpic;
ncnn::convert_packing(outpic,pakpic,3);
//
cv::Mat cvadj(160,160,CV_32FC3,pakpic.data);
//
float scale = 1.0f;
cvadj.convertTo(picreal.cvmat(),CV_8UC3,scale);
cv::Mat cout;
cv::cvtColor(picreal.cvmat(),cout,cv::COLOR_RGB2BGR);
BlendGramAlpha((uchar*)cout.data,(uchar*)mat_weights->data(),(uchar*)pic->data(),160,160);
//BlendGramAlpha((uchar*)picreal.udata(),(uchar*)mat_weights->data(),(uchar*)pic->data(),160,160);
//cv::cvtColor(picreal.cvmat(),pic->cvmat(),cv::COLOR_RGB2BGR);
*/
return 0;
}
int Mobunet::preprocess(JMat* pic,JMat* feat){
//pic 168
cv::Mat roipic(pic->cvmat(),cv::Rect(4,4,160,160));
JMat picmask(160,160,3,0,1);
JMat picreal(160,160,3,0,1);
cv::Mat cvmask = picmask.cvmat();
cv::Mat cvreal = picreal.cvmat();
roipic.copyTo(cvmask);
roipic.copyTo(cvreal);
cv::rectangle(cvmask,cv::Rect(5,5,150,145),cv::Scalar(0,0,0),-1);//,cv::LineTypes::FILLED);
domodel(&picreal,&picmask,feat);
cvreal.copyTo(roipic);
return 0;
}
int Mobunet::fgprocess(JMat* pic,const int* boxs,JMat* feat,JMat* fg){
int boxx, boxy ,boxwidth, boxheight ;
boxx = boxs[0];boxy=boxs[1];boxwidth=boxs[2]-boxx;boxheight=boxs[3]-boxy;
int stride = pic->stride();
cv::Mat roisrc(pic->cvmat(),cv::Rect(boxx,boxy,boxwidth,boxheight));
cv::Mat cvorig;
cv::resize(roisrc , cvorig, cv::Size(168, 168), cv::INTER_AREA);
JMat pic168(168,168,(uint8_t*)cvorig.data);
preprocess(&pic168,feat);
cv::Mat cvrst;;
cv::resize(cvorig , cvrst, cv::Size(boxwidth, boxheight), cv::INTER_AREA);
cv::Mat roidst(fg->cvmat(),cv::Rect(boxx,boxy,boxwidth,boxheight));
cvrst.copyTo(roidst);
return 0;
}
int Mobunet::process(JMat* pic,const int* boxs,JMat* feat){
int boxx, boxy ,boxwidth, boxheight ;
boxx = boxs[0];boxy=boxs[1];boxwidth=boxs[2]-boxx;boxheight=boxs[3]-boxy;
int stride = pic->stride();
cv::Mat roisrc(pic->cvmat(),cv::Rect(boxx,boxy,boxwidth,boxheight));
cv::Mat cvorig;
cv::resize(roisrc , cvorig, cv::Size(168, 168), cv::INTER_AREA);
JMat pic168(168,168,(uint8_t*)cvorig.data);
preprocess(&pic168,feat);
cv::Mat cvrst;;
cv::resize(cvorig , cvrst, cv::Size(boxwidth, boxheight), cv::INTER_AREA);
cvrst.copyTo(roisrc);
return 0;
}
int Mobunet::process2(JMat* pic,const int* boxs,JMat* feat){
int boxx, boxy ,boxwidth, boxheight ;
boxx = boxs[0];boxy=boxs[1];boxwidth=boxs[2]-boxx;boxheight=boxs[3]-boxy;
int stride = pic->stride();
cv::Mat cvsrc = pic->cvmat();
printf("cvsrc %d %d \n",cvsrc.cols,cvsrc.rows);
cv::Mat roisrc(cvsrc,cv::Rect(boxx,boxy,boxwidth,boxheight));
cv::Mat cvorig;
cv::resize(roisrc , cvorig, cv::Size(168, 168), cv::INTER_AREA);
/*
uint8_t* data =(uint8_t*)pic->data() + boxy*stride + boxx*pic->channel();
int scale_w = 168;
int scale_h = 168;
ncnn::Mat prepic = ncnn::Mat::from_pixels_resize(data, ncnn::Mat::PIXEL_BGR, boxwidth, boxheight, stride,scale_w, scale_h);
//pic 168
cv::Mat cvorig(168,168,CV_8UC3,prepic.data);
*/
cv::Mat roimask(cvorig,cv::Rect(4,4,160,160));
JMat picmask(160,160,3,0,1);
JMat picreal(160,160,3,0,1);
cv::Mat cvmask = picmask.cvmat();
cv::Mat cvreal = picreal.cvmat();
roimask.copyTo(cvmask);
roimask.copyTo(cvreal);
cv::rectangle(cvmask,cv::Rect(5,5,150,150),cv::Scalar(0,0,0),-1);//,cv::LineTypes::FILLED);
// cv::imshow("000",cvorig);
// cv::imshow("aaa",cvmask);
// cv::imshow("bbb",cvreal);
// cv::waitKey(0);
ncnn::Mat inmask = ncnn::Mat::from_pixels(picmask.udata(), ncnn::Mat::PIXEL_BGR2RGB, 160, 160);
inmask.substract_mean_normalize(mean_vals, norm_vals);
ncnn::Mat inreal = ncnn::Mat::from_pixels(picreal.udata(), ncnn::Mat::PIXEL_BGR2RGB, 160, 160);
inreal.substract_mean_normalize(mean_vals, norm_vals);
JMat picin(160*160,2,3);
char* pd = (char*)picin.data();
memcpy(pd,inreal.data,160*160*3*4);
memcpy(pd+ 160*160*3*4,inmask.data,160*160*3*4);
// char* pinpic = NULL;
// dumpfile("pic.bin",&pinpic);
// dumpfloat((float*)pd,10);
// dumpfloat((float*)pinpic,10);
//ncnn::Mat inpic(160,160,6,pd,4);
ncnn::Mat inpack(160,160,1,pd,(size_t)4u*6,6);
ncnn::Mat inpic;
ncnn::convert_packing(inpack,inpic,1);
// char* pwenet = NULL;
// dumpfile("wenet.bin",&pwenet);
ncnn::Mat inwenet(256,20,1,feat->data(),4);
ncnn::Mat outpic;
ncnn::Extractor ex = unet.create_extractor();
ex.input("face", inpic);
ex.input("audio", inwenet);
ex.extract("output", outpic);
float outmean_vals[3] = {-1.0f, -1.0f, -1.0f};
// float outnorm_vals[3] = { 2.0f, 2.0f, 2.0f};
float outnorm_vals[3] = { 127.5f, 127.5f, 127.5f};
outpic.substract_mean_normalize(outmean_vals, outnorm_vals);
ncnn::Mat pakpic;
ncnn::convert_packing(outpic,pakpic,3);
cv::Mat cvadj(160,160,CV_32FC3,pakpic.data);
cv::Mat cvout(160,160,CV_8UC3);
float scale = 1.0f;
cvadj.convertTo(cvout,CV_8UC3,scale);
//cv::imwrite("cvout.jpg",cvout);
cv::cvtColor(cvout,roimask,cv::COLOR_RGB2BGR);
// cvout.copyTo(roimask);
//cv::imwrite("roimask.jpg",roimask);
//cv::imwrite("cvorig.jpg",cvorig);
//cv::waitKey(0);
cv::resize(cvorig , roisrc, cv::Size(boxwidth, boxheight), cv::INTER_AREA);
//cv::imwrite("roisrc.jpg",roisrc);
//cv::imshow("cvsrc",cvsrc);
// cv::imshow("roisrc",roisrc);
// cv::imshow("cvorig",cvorig);
// cv::waitKey(20);
/*
{
uint8_t *pr = (uint8_t *) cvoutc.data;
printf("==%u %u %u\n", pr[0], pr[1], pr[2]);
}
//
float* p = (float*)cvadj.data;
printf("==%f %f %f\n",p[0],p[1],p[2]);
p+=160*160;
printf("==%f %f %f\n",p[0],p[1],p[2]);
p+=160*160;
printf("==%f %f %f\n",p[0],p[1],p[2]);
*/
return 0;
}

View File

@@ -0,0 +1,28 @@
#pragma once
#include "jmat.h"
#include "net.h"
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <stdio.h>
#include <vector>
class Mobunet{
private:
ncnn::Net unet;
float mean_vals[3] = {127.5f, 127.5f, 127.5f};
float norm_vals[3] = {1 / 127.5f, 1 / 127.5f, 1 / 127.5f};
JMat* mat_weights = nullptr;
int initModel(const char* binfn,const char* paramfn,const char* mskfn);
public:
int domodel(JMat* pic,JMat* msk,JMat* feat);
int domodelold(JMat* pic,JMat* msk,JMat* feat);
int preprocess(JMat* pic,JMat* feat);
int process(JMat* pic,const int* boxs,JMat* feat);
int fgprocess(JMat* pic,const int* boxs,JMat* feat,JMat* fg);
int process2(JMat* pic,const int* boxs,JMat* feat);
Mobunet(const char* modeldir,const char* modelid);
Mobunet(const char* fnbin,const char* fnparam,const char* fnmsk);
~Mobunet();
};

View File

@@ -0,0 +1,272 @@
#include "netwav.h"
#include "face_utils.h"
#include "jlog.h"
#include "wavreader.h"
int KWav::initbuf(int pcmsample){
m_pcmsample = pcmsample;
m_wavsample = pcmsample + 2*MFCC_OFFSET;
m_seca = m_wavsample / MFCC_WAVCHUNK;
m_secb = m_wavsample % MFCC_WAVCHUNK;
m_mellast = m_secb?(m_secb /160 +1):0;
m_bnflast = m_secb?((m_mellast*0.25f)-0.75f):0;
m_wavsize = m_seca*MFCC_WAVCHUNK + m_secb;
m_melsize = m_seca*MFCC_MELBASE+m_mellast;
m_bnfsize = m_seca*MFCC_BNFBASE+m_bnflast;
//m_calcsize = m_seca+m_secb?1:0;
m_calcsize = m_seca+(m_secb?1:0);
m_wavmat = new JMat(MFCC_WAVCHUNK,m_calcsize,1);
m_wavmat->zeros();
m_melmat = new JMat(MFCC_MELCHUNK*MFCC_MELBASE,m_calcsize,1);
m_melmat->zeros();
//m_bnfmat = new JMat(MFCC_BNFCHUNK*MFCC_BNFBASE,m_calcsize,1);
//m_bnfmat->zeros();
m_bnfblock = m_duration*MFCC_FPS;
if(m_bnfblock>(m_bnfsize-10))m_bnfblock = m_bnfsize-10;
LOGD("==seca %d secb %d\n",m_seca,m_secb);
LOGD("==melsize %d bnfsize %d\n",m_melsize,m_bnfsize);
return 0;
}
int KWav::initinx(){
m_curwav = m_wavmat->fdata() + MFCC_OFFSET;
m_leftsample = m_pcmsample;
m_waitsample = MFCC_OFFSET;
/*
int* arr = m_bnfmat->tagarr();
arr[0] = m_pcmsample*2;
arr[1] = m_pcmsample;
arr[2] = m_seca;
arr[3] = m_secb;
arr[4] = m_melsize;
arr[5] = m_bnfsize;
*/
float secs = m_pcmsample*1.0f/ MFCC_RATE;
int bnfblock = secs*MFCC_FPS;
if(bnfblock>(m_bnfsize-10))bnfblock = m_bnfsize-10;
//arr[6] = bnfblock;//m_bnfblock;
//
//LOGD("my==bnfblock %d arr6 %d",bnfblock,arr[6]);
//LOGD("myarr %d %d %d %d %d %d %d",arr[0],arr[1],arr[2],arr[3],arr[4],arr[5],arr[6]);
return 0;
}
int KWav::incsample(int sample){
if(sample<1)return 0;
m_leftsample -= sample;
m_waitsample += sample;
//LOGD("===incsample %d left %d wait %d\n",sample,m_leftsample,m_waitsample);
while(m_waitsample>MFCC_WAVCHUNK){
m_waitsample -= MFCC_WAVCHUNK;
m_waitcnt += 1;
LOGD("===tooken calc %d waitcnt %d calc %d\n",m_calcsize,m_waitcnt,m_calccnt);
LOGD("===tooken m_ldftsample %d\n",m_leftsample);
}
if(m_leftsample<=0){
m_waitcnt = m_calcsize;
LOGD("===tooken m_ldftsample %d\n",m_leftsample);
LOGD("===tooken calc %d waitcnt %d\n",m_calcsize,m_waitcnt);
}
return 0;
}
int KWav::pushpcm(uint8_t* pcm,int size){
uint8_t* pstart = pcm;
int psize = size;
if(m_alonecnt){
pstart++;
psize--;
m_alonearr[1]=*pcm;
float* ps = (float*)m_alonearr;
*m_curwav++ = (float)(*ps++/32767.f);
incsample(1);
m_alonecnt = 0;
}
int sample = psize / 2;
//LOGD("push pcm %d left_sample %d\n",sample,m_leftsample);
int left = psize % 2;
if(sample>m_leftsample){
sample = m_leftsample;
left = 0;
}
short* ps = (short*)pstart;
float* pf = m_curwav;
for(int k=0;k<sample;k++){
*pf++ = (float)(*ps++/32767.f);
}
m_curwav = pf;
//LOGD("push pcm %d\n",sample);
incsample(sample);
if(left){
uint8_t* pc = (uint8_t*)ps;
m_alonearr[0] = *pc;
m_alonecnt = 1;
}
return sample;
}
/*
int KWav::loadfn(const char* wavfile){
int format, channels, sr, bits_per_sample;
unsigned int data_length;
void* fhnd = wav_read_open(wavfile);
if(!fhnd)return -1;
int res = wav_get_header(fhnd, &format, &channels, &sr, &bits_per_sample, &data_length);
if(data_length<1) return -2;
//init
initbuf(data_length/2);
//
wav_read_close(fhnd);
return 0;
}
*/
KWav::KWav(const char* filename,MBnfCache* bnfcache){
m_bnfcache = bnfcache;
std::string wavfile(filename);
int format, channels, sr, bits_per_sample;
unsigned int data_length;
void* fhnd = wav_read_open(wavfile.c_str());
if(!fhnd){
m_duration = 0;
return;
}
int res = wav_get_header(fhnd, &format, &channels, &sr, &bits_per_sample, &data_length);
if(data_length<1) {
m_duration = 0;
return;
}
int sample = data_length/2;
m_duration = sample*1.0f/16000.0f;
initbuf(sample);
initinx();
JBuf* pcmbuf = new JBuf(data_length);
int rst = wav_read_data(fhnd,(unsigned char*)pcmbuf->data(),data_length);
//
short* ps = (short*)pcmbuf->data();
float* pd = (float*)m_wavmat->data();
float* pf = pd+MFCC_OFFSET;
for(int k=0;k<sample;k++){
*pf++ = (float)(*ps++/ 32767.f);
}
incsample(sample);
readyall();
delete pcmbuf;
}
KWav::KWav(float duration,MBnfCache* bnfcache){
m_bnfcache = bnfcache;
m_duration = duration;
int sample = duration*16000;
initbuf(sample);
initinx();
}
KWav::~KWav(){
if(m_wavmat){
delete m_wavmat;
m_wavmat = nullptr;
}
if(m_melmat){
delete m_melmat;
m_melmat = nullptr;
}
//if(m_bnfmat){
//delete m_bnfmat;
//m_bnfmat = nullptr;
//}
}
int KWav::bnfblocks(){
return m_bnfblock;
}
/*
JMat* KWav::bnfmat(){
return m_bnfmat;
}
*/
int KWav::finishone(int index){
m_resultcnt = index;
//int* arr = m_bnfmat->tagarr();
float secs = index*MFCC_WAVCHUNK *1.0f/ MFCC_RATE;
if(m_resultcnt==m_calccnt){
secs = m_pcmsample*1.0f/ MFCC_RATE;
}else{
secs = index*MFCC_WAVCHUNK *1.0f/ MFCC_RATE;
}
int bnfblock = secs*MFCC_FPS;
if(bnfblock>(m_bnfsize-10))bnfblock = m_bnfsize-10;
//arr[7] = bnfblock;
return 0;
}
int KWav::isfinish(){
if((m_waitcnt == m_calcsize)&&(m_calcsize== m_calccnt)){
return m_resultcnt==m_calccnt;
}else{
return 0;
}
}
int KWav::readyall(){
m_waitcnt=m_calcsize;
return 0;
}
int KWav::isready(){
if(m_waitcnt>m_calccnt){
return ++m_calccnt;
}else{
return 0;
}
}
int KWav::calcbuf(int calcinx,float** ppwav,float** ppmfcc,float** ppbnf,int* pmel,int* pbnf){
if(calcinx>m_calcsize)return -1;
if(calcinx<1)return -2;
int index = calcinx -1;
//LOGD("===tooken calcbuf %d\n",index);
*ppwav = m_wavmat->frow(index);
*ppmfcc = m_melmat->frow(index);
//*ppbnf = m_bnfmat->frow(index);
*ppbnf = m_bnfcache->secBuf(index)->fdata();
if(calcinx==m_calcsize){
*pmel = m_mellast;
*pbnf = m_bnflast;
}else{
*pmel = MFCC_MELBASE;
*pbnf = MFCC_BNFBASE;
}
return calcinx;
}
float KWav::duration(){
return m_duration;
}
int KWav::resultcnt(){
return m_resultcnt;
}
int KWav::debug(){
//dumpfloat(m_bnfmat->fdata(),10);
return 0;
}

View File

@@ -0,0 +1,56 @@
#pragma once
#include "jmat.h"
#include "aicommon.h"
#include "wavcache.h"
class KWav{
private:
float m_duration = 0;
int m_pcmsample = 0;;
int m_wavsample = 0;;
int m_seca = 0;
int m_secb = 0;
int m_mellast = 0;
int m_bnflast = 0;
int m_wavsize = 0;
int m_melsize = 0;
int m_bnfsize = 0;
int m_calcsize = 0;
int m_bnfblock = 0;
uint8_t m_alonearr[2] ;
int m_alonecnt = 0;
int m_leftsample = 0;
float *m_curwav = nullptr;
int m_waitsample = 0;
int m_waitcnt = 0;
int m_calccnt = 0;
int m_resultcnt = 0;
int incsample(int sample);
JMat *m_wavmat = nullptr;
JMat *m_melmat = nullptr;
//JMat *m_bnfmat = nullptr;
MBnfCache *m_bnfcache = nullptr;
int initbuf(int pcmsample);
int initinx();
public:
KWav(float duration,MBnfCache* bnfcache);
KWav(const char* filename,MBnfCache* bnfcache);
//KWav(const char* wavfn);
~KWav();
int pushpcm(uint8_t* pcm,int size);
int isready();
int readyall();
int isfinish();
int finishone(int index);
int bnfblocks();
float duration();
int resultcnt();
//JMat* bnfmat();
int calcbuf(int calcinx,float** ppwav,float** ppmfcc,float** ppbnf,int* pmel,int* pbnf);
int debug();
};

View File

@@ -0,0 +1,200 @@
#include "pfpld.h"
#include "cpu.h"
static int pts68_pfpld(float* arr_pts98,float* arr_pts68){
float* arr = arr_pts98;
float* dst = arr_pts68;
for(int j=0;j<17;j++){
*dst++ = arr[j*4];
*dst++ = arr[j*4 + 1];
}
for(int j=33;j<38;j++){
*dst++ = arr[j*2];
*dst++ = arr[j*2 + 1];
}
for(int j=42;j<47;j++){
*dst++ = arr[j*2];
*dst++ = arr[j*2 + 1];
}
for(int j=51;j<61;j++){
*dst++ = arr[j*2];
*dst++ = arr[j*2 + 1];
}
float* points = arr;
float point_38_x = (float(points[60 * 2 + 0]) + float(points[62 * 2 + 0])) / 2.0;
float point_38_y = (float(points[60 * 2 + 1]) + float(points[62 * 2 + 1])) / 2.0;
float point_39_x = (float(points[62 * 2 + 0]) + float(points[64 * 2 + 0])) / 2.0;
float point_39_y = (float(points[62 * 2 + 1]) + float(points[64 * 2 + 1])) / 2.0;
float point_41_x = (float(points[64 * 2 + 0]) + float(points[66 * 2 + 0])) / 2.0;
float point_41_y = (float(points[64 * 2 + 1]) + float(points[66 * 2 + 1])) / 2.0;
float point_42_x = (float(points[60 * 2 + 0]) + float(points[66 * 2 + 0])) / 2.0;
float point_42_y = (float(points[60 * 2 + 1]) + float(points[66 * 2 + 1])) / 2.0;
float point_44_x = (float(points[68 * 2 + 0]) + float(points[70 * 2 + 0])) / 2.0;
float point_44_y = (float(points[68 * 2 + 1]) + float(points[70 * 2 + 1])) / 2.0;
float point_45_x = (float(points[70 * 2 + 0]) + float(points[72 * 2 + 0])) / 2.0;
float point_45_y = (float(points[70 * 2 + 1]) + float(points[72 * 2 + 1])) / 2.0;
float point_47_x = (float(points[72 * 2 + 0]) + float(points[74 * 2 + 0])) / 2.0;
float point_47_y = (float(points[72 * 2 + 1]) + float(points[74 * 2 + 1])) / 2.0;
float point_48_x = (float(points[68 * 2 + 0]) + float(points[74 * 2 + 0])) / 2.0;
float point_48_y = (float(points[68 * 2 + 1]) + float(points[74 * 2 + 1])) / 2.0;
*dst++ = point_38_x;
*dst++ = point_38_y;
*dst++ = point_39_x;
*dst++ = point_39_y;
*dst++ = points[64 * 2 + 0];
*dst++ = points[64 * 2 + 1];
*dst++ = point_41_x;
*dst++ = point_41_y;
*dst++ = point_42_x;
*dst++ = point_42_y;
*dst++ = points[68 * 2 + 0];
*dst++ = points[68 * 2 + 1];
*dst++ = point_44_x;
*dst++ = point_44_y;
*dst++ = point_45_x;
*dst++ = point_45_y;
*dst++ = points[72 * 2 + 0];
*dst++ = points[72 * 2 + 1];
*dst++ = point_47_x;
*dst++ = point_47_y;
*dst++ = point_48_x;
*dst++ = point_48_y;
for(int j = 76; j<96;j++){
*dst++ = points[j * 2 + 0];
*dst++ = points[j * 2 + 1];
}
int len = dst-arr_pts68;
printf("====%d\n",len);
return len;
}
int Pfpld::detect(JMat* pic,int* arrboxs,int* arrlands){
int w = arrboxs[0];
int h = arrboxs[1];
int boxx, boxy ,boxwidth, boxheight ;
int* boxs = arrboxs+6;
boxx = boxs[0];boxy=boxs[1];boxwidth=boxs[2]-boxx;boxheight=boxs[3]-boxy;
printf("===pfpld %d %d %d %d\n",boxx,boxy,boxwidth,boxheight);
//boxx=192; boxy =245 ; boxwidth=908 -boxx;boxheight=1203 - boxy;
int stride = pic->stride();
uint8_t* data =(uint8_t*)pic->data() + boxy*stride + boxx*pic->channel();
ncnn::Mat in = ncnn::Mat::from_pixels_resize(data, ncnn::Mat::PIXEL_BGR2RGB, boxwidth, boxheight, stride,scale_w, scale_h);
in.substract_mean_normalize(mean_vals, norm_vals);
ncnn::Extractor ex = pfpld.create_extractor();
ex.input("input", in);
ncnn::Mat pose_blob, landms_blob;
ex.extract("pose", pose_blob);
ex.extract("landms", landms_blob);
float* arr = (float*)landms_blob.data;
float tmpdst[512];
float* dst = tmpdst;
pts68_pfpld(arr,dst);
int len = 68*2;
int* apts = arrlands;
int axmin = 10000,aymin = 10000,axmax = 0,aymax = 0;
for(int j=0;j<len/2;j++){
float x = *dst++ * boxwidth + boxx;
float y = *dst++ * boxheight + boxy;
apts[0] = x;
apts[1] = y;
printf("==adj %d x %f y %f\n",j,x,y);
if(axmin>x) axmin = x;
if(axmax<x) axmax = x;
if(aymin>y) aymin = y;
if(aymax<y) aymax = y;
apts += 2;
}
printf("===all xmin %d ymin %d xmax %d ymax %d\n",axmin,aymin,axmax,aymax);
m_wh = (axmax-axmin)*1.0f/(aymax-aymin);
if(m_wh>1.0f)m_wh = 1.0f;
printf("====wh %f\n",m_wh);
float wh = m_wh;//0.8878923766816144;
int xmin = 10000,ymin = 10000,xmax = 0,ymax = 0;
apts = arrlands+2;
for(int j=1;j<16;j++){
int x = *apts++;
int y = *apts++;
if(xmin>x) xmin = x;
if(xmax<x) xmax = x;
if(ymin>y) ymin = y;
if(ymax<y) ymax = y;
printf("==min one %d x %d y %d\n",j,x,y);
}
printf("===b1 xmin %d ymin %d xmax %d ymax %d\n",xmin,ymin,xmax,ymax);
apts = arrlands+30*2;
for(int j=31;j<68;j++){
int x = *apts++;
int y = *apts++;
if(xmin>x) xmin = x;
if(xmax<x) xmax = x;
if(ymin>y) ymin = y;
if(ymax<y) ymax = y;
printf("==min two %d x %d y %d\n",j,x,y);
}
printf("===xmin %d ymin %d xmax %d ymax %d\n",xmin,ymin,xmax,ymax);
int rw = xmax-xmin;
int rh = ymax-ymin;
int x_c = xmin + rw/2;
float x1,x2,y1,y2;
x1 = x_c - rw*0.5f/wh;
x2 = x_c + rw*0.5f/wh;
y1 = ymin + rw*0.11f/wh;
y2 = ymin + rw*1.11f/wh;
if(x1<0)x1=0;
if(y1<0)y1=0;
if(x2>=m_width)x2=m_width-1;
if(y2>=m_height)y2=m_height-1;
boxs = arrboxs+10;
*boxs++ = x1;
*boxs++ = y1;
*boxs++ = x2;
*boxs++ = y2;
return 0;
}
void Pfpld::recal(int w,int h){
m_width = w;
m_height = h;
}
Pfpld::Pfpld(const char* modeldir,const char* modelid,int w,int h){
pfpld.clear();
ncnn::set_cpu_powersave(2);
ncnn::set_omp_num_threads(ncnn::get_big_cpu_count());
pfpld.opt = ncnn::Option();
//pfpld.opt.use_vulkan_compute = true;
pfpld.opt.num_threads = ncnn::get_big_cpu_count();
//pfpld.load_param("model/pfpld.param");
//pfpld.load_model("model/pfpld.bin");
char filepath[1024];
sprintf(filepath,"%s/%s.param",modeldir,modelid);
pfpld.load_param(filepath);
sprintf(filepath,"%s/%s.bin",modeldir,modelid);
pfpld.load_model(filepath);
recal(w,h);
}
Pfpld::~Pfpld(){
}
#ifdef _PFPLD_MAIN_
int main(int argc,char** argv){
Pfpld* pfpld = new Pfpld(1080,1920);
std::string picfile("1.jpg");
JMat* pic = new JMat(picfile,1);
int* arr = pic->tagarr();
pfpld->detect(pic,arr+6,arr+64);
printf("precess to exit\n");
getchar();
return 0;
}
#endif

View File

@@ -0,0 +1,27 @@
#pragma once
#include "jmat.h"
#include "net.h"
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <stdio.h>
#include <vector>
class Pfpld{
private:
ncnn::Net pfpld;
JMat* m_mat112;
int scale_w = 112;
int scale_h = 112;
float mean_vals[3] = {0.f, 0.f, 0.f};
float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f};
int m_width;
int m_height;
float m_wh;
int m_cnt;
void recal(int w,int h);
public:
int detect(JMat* pic,int* arrboxs,int* arrlands);
Pfpld(const char* modeldir,const char* modelid,int w,int h);
~Pfpld();
};

View File

@@ -0,0 +1,487 @@
#include "scrfd.h"
#include "cpu.h"
static int drawface(cv::Mat& rgb, const std::vector<FaceObject>& faceobjects)
{
int has_kps = 1;
for (size_t i = 0; i < faceobjects.size(); i++)
{
const FaceObject& obj = faceobjects[i];
// fprintf(stderr, "%.5f at %.2f %.2f %.2f x %.2f\n", obj.prob,
// obj.rect.x, obj.rect.y, obj.rect.width, obj.rect.height);
cv::rectangle(rgb, obj.rect, cv::Scalar(0, 255, 0));
if (has_kps)
{
cv::circle(rgb, obj.landmark[0], 2, cv::Scalar(255, 255, 0), -1);
cv::circle(rgb, obj.landmark[1], 2, cv::Scalar(255, 255, 0), -1);
cv::circle(rgb, obj.landmark[2], 2, cv::Scalar(255, 255, 0), -1);
cv::circle(rgb, obj.landmark[3], 2, cv::Scalar(255, 255, 0), -1);
cv::circle(rgb, obj.landmark[4], 2, cv::Scalar(255, 255, 0), -1);
}
char text[256];
sprintf(text, "%.1f%%", obj.prob * 100);
int baseLine = 0;
cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
int x = obj.rect.x;
int y = obj.rect.y - label_size.height - baseLine;
if (y < 0)
y = 0;
if (x + label_size.width > rgb.cols)
x = rgb.cols - label_size.width;
cv::rectangle(rgb, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), cv::Scalar(255, 255, 255), -1);
cv::putText(rgb, text, cv::Point(x, y + label_size.height), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 0, 0), 1);
}
return 0;
}
static inline float intersection_area(const FaceObject& a, const FaceObject& b)
{
cv::Rect_<float> inter = a.rect & b.rect;
return inter.area();
}
static void qsort_descent_inplace(std::vector<FaceObject>& faceobjects, int left, int right)
{
int i = left;
int j = right;
float p = faceobjects[(left + right) / 2].prob;
while (i <= j)
{
while (faceobjects[i].prob > p)
i++;
while (faceobjects[j].prob < p)
j--;
if (i <= j)
{
// swap
std::swap(faceobjects[i], faceobjects[j]);
i++;
j--;
}
}
// #pragma omp parallel sections
{
// #pragma omp section
{
if (left < j) qsort_descent_inplace(faceobjects, left, j);
}
// #pragma omp section
{
if (i < right) qsort_descent_inplace(faceobjects, i, right);
}
}
}
static void qsort_descent_inplace(std::vector<FaceObject>& faceobjects)
{
if (faceobjects.empty())
return;
qsort_descent_inplace(faceobjects, 0, faceobjects.size() - 1);
}
static void nms_sorted_bboxes(const std::vector<FaceObject>& faceobjects, std::vector<int>& picked, float nms_threshold)
{
picked.clear();
const int n = faceobjects.size();
std::vector<float> areas(n);
for (int i = 0; i < n; i++)
{
areas[i] = faceobjects[i].rect.area();
}
for (int i = 0; i < n; i++)
{
const FaceObject& a = faceobjects[i];
int keep = 1;
for (int j = 0; j < (int)picked.size(); j++)
{
const FaceObject& b = faceobjects[picked[j]];
// intersection over union
float inter_area = intersection_area(a, b);
float union_area = areas[i] + areas[picked[j]] - inter_area;
// float IoU = inter_area / union_area
if (inter_area / union_area > nms_threshold)
keep = 0;
}
if (keep)
picked.push_back(i);
}
}
// insightface/detection/scrfd/mmdet/core/anchor/anchor_generator.py gen_single_level_base_anchors()
static ncnn::Mat generate_anchors(int base_size, const ncnn::Mat& ratios, const ncnn::Mat& scales)
{
int num_ratio = ratios.w;
int num_scale = scales.w;
ncnn::Mat anchors;
anchors.create(4, num_ratio * num_scale);
const float cx = 0;
const float cy = 0;
for (int i = 0; i < num_ratio; i++)
{
float ar = ratios[i];
int r_w = round(base_size / sqrt(ar));
int r_h = round(r_w * ar); //round(base_size * sqrt(ar));
for (int j = 0; j < num_scale; j++)
{
float scale = scales[j];
float rs_w = r_w * scale;
float rs_h = r_h * scale;
float* anchor = anchors.row(i * num_scale + j);
anchor[0] = cx - rs_w * 0.5f;
anchor[1] = cy - rs_h * 0.5f;
anchor[2] = cx + rs_w * 0.5f;
anchor[3] = cy + rs_h * 0.5f;
}
}
return anchors;
}
static void generate_proposals(const ncnn::Mat& anchors, int feat_stride, const ncnn::Mat& score_blob, const ncnn::Mat& bbox_blob, const ncnn::Mat& kps_blob, float prob_threshold, std::vector<FaceObject>& faceobjects)
{
int w = score_blob.w;
int h = score_blob.h;
// generate face proposal from bbox deltas and shifted anchors
const int num_anchors = anchors.h;
for (int q = 0; q < num_anchors; q++)
{
const float* anchor = anchors.row(q);
const ncnn::Mat score = score_blob.channel(q);
const ncnn::Mat bbox = bbox_blob.channel_range(q * 4, 4);
// shifted anchor
float anchor_y = anchor[1];
float anchor_w = anchor[2] - anchor[0];
float anchor_h = anchor[3] - anchor[1];
for (int i = 0; i < h; i++)
{
float anchor_x = anchor[0];
for (int j = 0; j < w; j++)
{
int index = i * w + j;
float prob = score[index];
if (prob >= prob_threshold)
{
// insightface/detection/scrfd/mmdet/models/dense_heads/scrfd_head.py _get_bboxes_single()
float dx = bbox.channel(0)[index] * feat_stride;
float dy = bbox.channel(1)[index] * feat_stride;
float dw = bbox.channel(2)[index] * feat_stride;
float dh = bbox.channel(3)[index] * feat_stride;
// insightface/detection/scrfd/mmdet/core/bbox/transforms.py distance2bbox()
float cx = anchor_x + anchor_w * 0.5f;
float cy = anchor_y + anchor_h * 0.5f;
float x0 = cx - dx;
float y0 = cy - dy;
float x1 = cx + dw;
float y1 = cy + dh;
FaceObject obj;
obj.rect.x = x0;
obj.rect.y = y0;
obj.rect.width = x1 - x0 + 1;
obj.rect.height = y1 - y0 + 1;
obj.prob = prob;
if (!kps_blob.empty())
{
const ncnn::Mat kps = kps_blob.channel_range(q * 10, 10);
obj.landmark[0].x = cx + kps.channel(0)[index] * feat_stride;
obj.landmark[0].y = cy + kps.channel(1)[index] * feat_stride;
obj.landmark[1].x = cx + kps.channel(2)[index] * feat_stride;
obj.landmark[1].y = cy + kps.channel(3)[index] * feat_stride;
obj.landmark[2].x = cx + kps.channel(4)[index] * feat_stride;
obj.landmark[2].y = cy + kps.channel(5)[index] * feat_stride;
obj.landmark[3].x = cx + kps.channel(6)[index] * feat_stride;
obj.landmark[3].y = cy + kps.channel(7)[index] * feat_stride;
obj.landmark[4].x = cx + kps.channel(8)[index] * feat_stride;
obj.landmark[4].y = cy + kps.channel(9)[index] * feat_stride;
}
faceobjects.push_back(obj);
}
anchor_x += feat_stride;
}
anchor_y += feat_stride;
}
}
}
int Scrfd::detect(JMat* pic,int* boxs){
recal(pic->width(),pic->height());
ncnn::Mat in = ncnn::Mat::from_pixels_resize(pic->udata(), ncnn::Mat::PIXEL_BGR2RGB, m_width, m_height, scale_w, scale_h);
ncnn::Mat in_pad;
ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, ncnn::BORDER_CONSTANT, 0.f);
in_pad.substract_mean_normalize(mean_vals, norm_vals);
int has_kps = 1;
ncnn::Extractor ex = scrfd.create_extractor();
ex.input("input.1", in_pad);
std::vector<FaceObject> faceobjects;
std::vector<FaceObject> faceproposals;
//std::vector<FaceObject>& faceobjects, float prob_threshold, float nms_threshold)
// stride 8
{
ncnn::Mat score_blob, bbox_blob, kps_blob;
ex.extract("score_8", score_blob);
ex.extract("bbox_8", bbox_blob);
if (has_kps)
ex.extract("kps_8", kps_blob);
const int base_size = 16;
const int feat_stride = 8;
ncnn::Mat ratios(1);
ratios[0] = 1.f;
ncnn::Mat scales(2);
scales[0] = 1.f;
scales[1] = 2.f;
ncnn::Mat anchors = generate_anchors(base_size, ratios, scales);
std::vector<FaceObject> faceobjects32;
generate_proposals(anchors, feat_stride, score_blob, bbox_blob, kps_blob, prob_threshold, faceobjects32);
faceproposals.insert(faceproposals.end(), faceobjects32.begin(), faceobjects32.end());
}
// stride 16
{
ncnn::Mat score_blob, bbox_blob, kps_blob;
ex.extract("score_16", score_blob);
ex.extract("bbox_16", bbox_blob);
if (has_kps)
ex.extract("kps_16", kps_blob);
const int base_size = 64;
const int feat_stride = 16;
ncnn::Mat ratios(1);
ratios[0] = 1.f;
ncnn::Mat scales(2);
scales[0] = 1.f;
scales[1] = 2.f;
ncnn::Mat anchors = generate_anchors(base_size, ratios, scales);
std::vector<FaceObject> faceobjects16;
generate_proposals(anchors, feat_stride, score_blob, bbox_blob, kps_blob, prob_threshold, faceobjects16);
faceproposals.insert(faceproposals.end(), faceobjects16.begin(), faceobjects16.end());
}
// stride 32
{
ncnn::Mat score_blob, bbox_blob, kps_blob;
ex.extract("score_32", score_blob);
ex.extract("bbox_32", bbox_blob);
if (has_kps)
ex.extract("kps_32", kps_blob);
const int base_size = 256;
const int feat_stride = 32;
ncnn::Mat ratios(1);
ratios[0] = 1.f;
ncnn::Mat scales(2);
scales[0] = 1.f;
scales[1] = 2.f;
ncnn::Mat anchors = generate_anchors(base_size, ratios, scales);
std::vector<FaceObject> faceobjects8;
generate_proposals(anchors, feat_stride, score_blob, bbox_blob, kps_blob, prob_threshold, faceobjects8);
faceproposals.insert(faceproposals.end(), faceobjects8.begin(), faceobjects8.end());
}
// sort all proposals by score from highest to lowest
qsort_descent_inplace(faceproposals);
// apply nms with nms_threshold
std::vector<int> picked;
nms_sorted_bboxes(faceproposals, picked, nms_threshold);
int face_count = picked.size();
printf("====face_count %d\n",face_count);
faceobjects.resize(face_count);
for (int i = 0; i < face_count; i++)
{
faceobjects[i] = faceproposals[picked[i]];
// adjust offset to original unpadded
float x0 = (faceobjects[i].rect.x - (wpad / 2)) / scale;
float y0 = (faceobjects[i].rect.y - (hpad / 2)) / scale;
float x1 = (faceobjects[i].rect.x + faceobjects[i].rect.width - (wpad / 2)) / scale;
float y1 = (faceobjects[i].rect.y + faceobjects[i].rect.height - (hpad / 2)) / scale;
x0 = std::max(std::min(x0, (float)m_width - 1), 0.f);
y0 = std::max(std::min(y0, (float)m_height - 1), 0.f);
x1 = std::max(std::min(x1, (float)m_width - 1), 0.f);
y1 = std::max(std::min(y1, (float)m_height - 1), 0.f);
faceobjects[i].rect.x = x0;
faceobjects[i].rect.y = y0;
faceobjects[i].rect.width = x1 - x0;
faceobjects[i].rect.height = y1 - y0;
if (has_kps)
{
float x0 = (faceobjects[i].landmark[0].x - (wpad / 2)) / scale;
float y0 = (faceobjects[i].landmark[0].y - (hpad / 2)) / scale;
float x1 = (faceobjects[i].landmark[1].x - (wpad / 2)) / scale;
float y1 = (faceobjects[i].landmark[1].y - (hpad / 2)) / scale;
float x2 = (faceobjects[i].landmark[2].x - (wpad / 2)) / scale;
float y2 = (faceobjects[i].landmark[2].y - (hpad / 2)) / scale;
float x3 = (faceobjects[i].landmark[3].x - (wpad / 2)) / scale;
float y3 = (faceobjects[i].landmark[3].y - (hpad / 2)) / scale;
float x4 = (faceobjects[i].landmark[4].x - (wpad / 2)) / scale;
float y4 = (faceobjects[i].landmark[4].y - (hpad / 2)) / scale;
faceobjects[i].landmark[0].x = std::max(std::min(x0, (float)m_width - 1), 0.f);
faceobjects[i].landmark[0].y = std::max(std::min(y0, (float)m_height - 1), 0.f);
faceobjects[i].landmark[1].x = std::max(std::min(x1, (float)m_width - 1), 0.f);
faceobjects[i].landmark[1].y = std::max(std::min(y1, (float)m_height - 1), 0.f);
faceobjects[i].landmark[2].x = std::max(std::min(x2, (float)m_width - 1), 0.f);
faceobjects[i].landmark[2].y = std::max(std::min(y2, (float)m_height - 1), 0.f);
faceobjects[i].landmark[3].x = std::max(std::min(x3, (float)m_width - 1), 0.f);
faceobjects[i].landmark[3].y = std::max(std::min(y3, (float)m_height - 1), 0.f);
faceobjects[i].landmark[4].x = std::max(std::min(x4, (float)m_width - 1), 0.f);
faceobjects[i].landmark[4].y = std::max(std::min(y4, (float)m_height - 1), 0.f);
}
break;
}
if(faceobjects.size()>0){
FaceObject fo = faceobjects[0];
cv::Rect_<float> rect = fo.rect;
//std::cout<<fo.rect<<std::endl;
int w = rect.width;
int h = rect.height;
int x1 = rect.x;
int y1 = rect.y;
int x2 = rect.x+w;
int y2 = rect.y+h;
printf("scrfd x1 %d y1 %d x2 %d y2 %d\n",x1,y1,x2,y2);
int* tag = boxs;//pic->tagarr();
int inx = 0;
tag[inx++] = w;
tag[inx++] = h;
tag[inx++] = x1;
tag[inx++] = y1;
tag[inx++] = x2;
tag[inx++] = y2;
float adj = w*0.1f;
x1 = x1 - adj;
if(x1<0)x1=0;
if(y1<0)y1=0;
adj = (x2-x1)*0.1f;
x2 = x2 + adj;
if(x2>=m_width)x2=m_width-1;
adj = (y2-y1)*0.1f;
y2 = y2 + adj;
if(y2>=m_height)y2=m_height-1;
tag[inx++] = x1;
tag[inx++] = y1;
tag[inx++] = x2;
tag[inx++] = y2;
printf("adj x1 %d y1 %d x2 %d y2 %d\n",x1,y1,x2,y2);
}
//cv::Mat drawmat(m_height,m_width,CV_8UC3,pic->udata());
//drawface(drawmat,faceobjects);
//cv::imshow("aaa",drawmat);
//cv::waitKey(0);
return 0;
}
void Scrfd::recal(int nw ,int nh){
if((nw==m_width)&&(nh==m_height))return;
m_width = nw;
m_height = nh;
int w = m_width;
int h = m_height;
scale = 1.f;
if (w > h) {
scale = (float)target_size / w;
w = target_size;
h = h * scale;
} else {
scale = (float)target_size / h;
h = target_size;
w = w * scale;
}
scale_h = h;
scale_w = w;
wpad = (w + 31) / 32 * 32 - w;
hpad = (h + 31) / 32 * 32 - h;
}
Scrfd::Scrfd(const char* modeldir,const char* modelid,int cols,int rows){
scrfd.clear();
ncnn::set_cpu_powersave(2);
ncnn::set_omp_num_threads(ncnn::get_big_cpu_count());
scrfd.opt = ncnn::Option();
//scrfd.opt.use_vulkan_compute = true;
scrfd.opt.num_threads = ncnn::get_big_cpu_count();
char filepath[1024];
sprintf(filepath,"%s/%s.param",modeldir,modelid);
scrfd.load_param(filepath);
sprintf(filepath,"%s/%s.bin",modeldir,modelid);
scrfd.load_model(filepath);
//scrfd.load_param("model/scrfd_500m_kps-opt2.param");
//scrfd.load_model("model/scrfd_500m_kps-opt2.bin");
//scrfd.load_param("model/scrfd.param");
//scrfd.load_model("model/scrfd.bin");
recal(cols,rows);
}
Scrfd::~Scrfd(){
}
#ifdef _SCRFD_MAIN_
int main(int argc,char** argv){
Scrfd* scrfd = new Scrfd(1080,1920);
std::string picfile("1.jpg");
JMat* pic = new JMat(picfile,1);
scrfd->detect(pic,pic->tagarr());
printf("precess to exit\n");
getchar();
return 0;
}
#endif

View File

@@ -0,0 +1,38 @@
#pragma once
#include "jmat.h"
#include "net.h"
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <stdio.h>
#include <vector>
struct FaceObject
{
cv::Rect_<float> rect;
cv::Point2f landmark[5];
float prob;
};
class Scrfd{
private:
ncnn::Net scrfd;
int target_size = 640;
float prob_threshold = 0.3f;
float nms_threshold = 0.45f;
float mean_vals[3] = {127.5f, 127.5f, 127.5f};
float norm_vals[3] = {1 / 128.f, 1 / 128.f, 1 / 128.f};
float scale;
int scale_w;
int scale_h;
int wpad;
int hpad;
int m_width;
int m_height;
void recal(int nw ,int nh);
public:
int detect(JMat* pic,int* boxs);
Scrfd(const char* modeldir,const char* modelid,int cols,int rows);
~Scrfd();
};

View File

@@ -0,0 +1,97 @@
#include "wavcache.h"
#include "aicommon.h"
JMat* MBufCache::secBuf(int sec){
JMat* mat = NULL;
m_lock->lock();
if(sec<vec_buf.size()){
mat = vec_buf[sec];
}else{
mat = new JMat(m_secw,m_sech+1,1);
vec_buf.push_back(mat);
}
m_lock->unlock();
return mat;
}
void MBufCache::debug(){
}
JMat* MBufCache::inxBuf(int inx){
int seca = inx/m_sech;
int secb = inx%m_sech;
JMat* mat = NULL;
if(secb>=m_lineh){
mat= new JMat(m_secw,m_blockh,1);
JMat* sa = secBuf(seca);
int la = m_sech-secb;
int parta = la*m_secw;
float* pa = mat->fdata();
memcpy(pa,sa->frow(secb),parta*sizeof(float));
JMat* sb = secBuf(seca+1);
int lb = m_blockh - la;
float* pb = pa+parta;
int partb = lb*m_secw;
memcpy(pb,sa->frow(0),partb*sizeof(float));
}else{
JMat* src = secBuf(seca);
float* buf = src->frow(secb);
//printf("==dist %d\n",(buf-src->fdata())*4);
mat = new JMat(m_secw,m_blockh,buf,1);
//printf("==size %d %d = %d\n",m_secw,m_blockh,m_secw*m_blockh*4);
}
return mat;
}
int* MBufCache::tagarr(){
return m_tagarr;
}
MBufCache::MBufCache(int initsec,int secw,int sech,int blockh){
m_lock = new std::mutex();
m_secw = secw;
m_sech = sech;
m_blockh = blockh;
m_lineh = sech-blockh;
for(int k=0;k<initsec;k++){
JMat* mat = new JMat(m_secw,m_sech+1,1);
vec_buf.push_back(mat);
}
memset(m_tagarr,0,512*sizeof(int));
}
MBufCache::~MBufCache(){
m_lock->lock();
for(int k=0;k<vec_buf.size();k++){
JMat* mat = vec_buf[k];
delete mat;
}
vec_buf.clear();
m_lock->unlock();
delete m_lock;
}
#include "aicommon.h"
MBnfCache::MBnfCache():MBufCache(3,MFCC_BNFCHUNK,MFCC_BNFBASE,20){
}
MBnfCache::~MBnfCache(){
}
#ifdef _WAVTEST_
int main(int argc,char** argv){
MBnfCache cache;
for(int k=0;k<100;k++){
JMat* mat = cache.secBuf(k);
}
for(int k=816;k<817;k++){
printf("#%d# \n",k);
JMat* mat = cache.inxBuf(k);
JMat cm = mat->clone();
delete mat;
}
return 0;
}
#endif

View File

@@ -0,0 +1,28 @@
#pragma once
#include "jmat.h"
#include <vector>
#include <mutex>
class MBufCache{
protected:
int m_lineh;
int m_secw;
int m_sech;
int m_blockh;
std::mutex *m_lock;
std::vector<JMat*> vec_buf ;
int m_tagarr[512];
public:
JMat* secBuf(int sec);
JMat* inxBuf(int inx);
int* tagarr();
MBufCache(int initsec,int secw,int sech,int blockh);
virtual ~MBufCache();
void debug();
};
class MBnfCache:public MBufCache{
public:
MBnfCache();
virtual ~MBnfCache();
};

View File

@@ -0,0 +1,210 @@
#include "wavreader.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#define TAG(a, b, c, d) (((a) << 24) | ((b) << 16) | ((c) << 8) | (d))
struct wav_reader {
FILE *wav;
long data_pos;
uint32_t data_length;
uint32_t data_left;
int format;
int sample_rate;
int bits_per_sample;
int channels;
int byte_rate;
int block_align;
int streamed;
};
static uint32_t read_tag(struct wav_reader* wr) {
uint32_t tag = 0;
char c1 = fgetc(wr->wav);
char c2 = fgetc(wr->wav);
char c3 = fgetc(wr->wav);
char c4 = fgetc(wr->wav);
tag = (tag << 8) | c1;
tag = (tag << 8) | c2;
tag = (tag << 8) | c3;
tag = (tag << 8) | c4;
printf("===tag %c %c %c %c\n",c1,c2,c3,c4);
return tag;
}
static uint32_t read_int32(struct wav_reader* wr) {
uint32_t value = 0;
value |= fgetc(wr->wav) << 0;
value |= fgetc(wr->wav) << 8;
value |= fgetc(wr->wav) << 16;
value |= fgetc(wr->wav) << 24;
return value;
}
static uint16_t read_int16(struct wav_reader* wr) {
uint16_t value = 0;
value |= fgetc(wr->wav) << 0;
value |= fgetc(wr->wav) << 8;
return value;
}
static void skip(FILE *f, int n) {
int i;
for (i = 0; i < n; i++)
fgetc(f);
}
void* wav_read_open(const char *filename) {
struct wav_reader* wr = (struct wav_reader*) malloc(sizeof(*wr));
memset(wr, 0, sizeof(*wr));
if (!strcmp(filename, "-"))
wr->wav = stdin;
else
wr->wav = fopen(filename, "rb");
if (wr->wav == NULL) {
free(wr);
return NULL;
}
while (1) {
uint32_t tag, tag2, length;
tag = read_tag(wr);
if (feof(wr->wav))
break;
length = read_int32(wr);
if (!length || length >= 0x7fff0000) {
wr->streamed = 1;
length = ~0;
}
if (tag != TAG('R', 'I', 'F', 'F') || length < 4) {
fseek(wr->wav, length, SEEK_CUR);
continue;
}
tag2 = read_tag(wr);
length -= 4;
if (tag2 != TAG('W', 'A', 'V', 'E')) {
fseek(wr->wav, length, SEEK_CUR);
continue;
}
// RIFF chunk found, iterate through it
while (length >= 8) {
uint32_t subtag, sublength;
subtag = read_tag(wr);
//char* fff = (char*)&subtag;
if (feof(wr->wav))
break;
sublength = read_int32(wr);
printf("==== subleng %d\n",sublength );
length -= 8;
if (length < sublength)
break;
if (subtag == TAG('f', 'm', 't', ' ')) {
if (sublength < 16) {
// Insufficient data for 'fmt '
break;
}
wr->format = read_int16(wr);
wr->channels = read_int16(wr);
wr->sample_rate = read_int32(wr);
wr->byte_rate = read_int32(wr);
wr->block_align = read_int16(wr);
wr->bits_per_sample = read_int16(wr);
if (wr->format == 0xfffe) {
if (sublength < 28) {
// Insufficient data for waveformatex
break;
}
skip(wr->wav, 8);
wr->format = read_int32(wr);
skip(wr->wav, sublength - 28);
}
else {
skip(wr->wav, sublength - 16);
}
}
else if (subtag == TAG('d', 'a', 't', 'a')) {
wr->data_pos = ftell(wr->wav);
wr->data_length = sublength;
printf("==== data subleng %d\n",sublength );
wr->data_left = wr->data_length;
if (!wr->data_length || wr->streamed) {
wr->streamed = 1;
return wr;
}
fseek(wr->wav, sublength, SEEK_CUR);
}
else {
skip(wr->wav, sublength);
}
length -= sublength;
}
if (length > 0) {
// Bad chunk?
fseek(wr->wav, length, SEEK_CUR);
}
}
fseek(wr->wav, wr->data_pos, SEEK_SET);
return wr;
}
void wav_read_close(void* obj) {
struct wav_reader* wr = (struct wav_reader*) obj;
if (wr->wav != stdin)
fclose(wr->wav);
free(wr);
}
int wav_get_header(void* obj, int* format, int* channels, int* sample_rate, int* bits_per_sample, unsigned int* data_length) {
struct wav_reader* wr = (struct wav_reader*) obj;
if (format)
*format = wr->format;
if (channels)
*channels = wr->channels;
if (sample_rate)
*sample_rate = wr->sample_rate;
if (bits_per_sample)
*bits_per_sample = wr->bits_per_sample;
printf("==== data left %d byterate %d\n",wr->data_left ,wr->byte_rate);
if (data_length)
*data_length = wr->data_length;
return wr->format && wr->sample_rate;
}
int wav_read_data(void* obj, unsigned char* data, unsigned int length) {
struct wav_reader* wr = (struct wav_reader*) obj;
int n;
if (wr->wav == NULL)
return -1;
if (length > wr->data_left && !wr->streamed) {
int loop = 1;
if (loop) {
fseek(wr->wav, wr->data_pos, SEEK_SET);
wr->data_left = wr->data_length;
}
length = wr->data_left;
}
n = fread(data, 1, length, wr->wav);
wr->data_left -= length;
return n;
}
#ifdef WAVTEST
int main(int argc,char** argv){
printf("====file %s\n",argv[1]);
void* hnd = wav_read_open(argv[1]);
int format;
int channels;
int sample_rate;
int bits_per_sample;
unsigned int data_length;
int rst = wav_get_header(hnd, &format, &channels, &sample_rate, &bits_per_sample, &data_length);
printf("===format %d channels %d sample_rate %d,bit %d,len %lu\n",format, channels, sample_rate, bits_per_sample, data_length);
return 0;
}
#endif

View File

@@ -0,0 +1,37 @@
/* ------------------------------------------------------------------
* Copyright (C) 2009 Martin Storsjo
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied.
* See the License for the specific language governing permissions
* and limitations under the License.
* -------------------------------------------------------------------
*/
#ifndef WAVREADER_H_
#define WAVREADER_H_
#ifdef __cplusplus
extern "C" {
#endif
void* wav_read_open(const char *filename);
void wav_read_close(void* obj);
int wav_get_header(void* obj, int* format, int* channels, int* sample_rate, int* bits_per_sample, unsigned int* data_length);
int wav_read_data(void* obj, unsigned char* data, unsigned int length);
#ifdef __cplusplus
}
#endif
#endif

View File

@@ -0,0 +1,196 @@
#include "wenet.h"
#include <stdio.h>
#include <memory>
#include <vector>
#include "wavreader.h"
#include "face_utils.h"
#include "mfcc/mfcc.hpp"
#include "jlog.h"
#include "aicommon.h"
void Wenet::initModel(const char* modelfn){
m_model = new OnnxModel();
string modelpath(modelfn);
m_model->initModel(modelpath);
//m_model->pushName("speech",1);
//m_model->pushName("speech_lengths",1);
//m_model->pushName("encoder_out",0);
}
int Wenet::calcbnf(float* melbin,int melnum,float* bnfbin,int bnfnum){
int rst = 0;
int chkmfcc = melnum;
int chkbnf = bnfnum;
auto onecfg = m_model->config();
auto cfg = &onecfg;
//cfg->dump();
cfg->shape_inputs[0][0] = 1;
cfg->shape_inputs[0][1] = chkmfcc;
cfg->size_inputs[0] = chkmfcc*MFCC_MELCHUNK;
cfg->shape_inputs[1][0] = 1;
cfg->size_inputs[1] = 1;
cfg->shape_outputs[0][0] = 1;
cfg->shape_outputs[0][1] = chkbnf;
cfg->shape_outputs[0][2] = MFCC_BNFCHUNK;
cfg->size_outputs[0] = chkbnf*MFCC_BNFCHUNK;
cfg->dump();
void* arrin[] = { melbin,&chkmfcc,NULL };
void* arrout[] = { bnfbin,NULL };
const char* namein[] = {"speech","speech_lengths",NULL};
const char* nameout[] = {"encoder_out",NULL};
cfg->names_in = namein;
cfg->names_out = nameout;
rst = m_model->runModel(arrin,arrout,NULL,cfg);
return rst;
}
Wenet::Wenet(const char* modeldir,const char* modelid){
char path[1024];
sprintf(path,"%s/%s.onnx",modeldir,modelid);
initModel((const char*)path);
}
Wenet::Wenet(const char* modelfn){
initModel(modelfn);
}
Wenet::~Wenet(){
delete m_model;
}
//int Wenet::nextwav(const char* wavfile,JMat** pmat){
int Wenet::nextwav(const char* wavfile,MBnfCache* bnfcache){
int m_pcmsample = 0;
JBuf *m_pcmbuf = nullptr;
JMat *m_wavmat = nullptr;
JMat *m_melmat = nullptr;
//JMat *m_bnfmat = nullptr;
int format, channels, sr, bits_per_sample;
unsigned int data_length;
void* fhnd = wav_read_open(wavfile);
if(!fhnd)return -1;
int res = wav_get_header(fhnd, &format, &channels, &sr, &bits_per_sample, &data_length);
if(data_length<1) {
wav_read_close(fhnd);
return -2;
}
LOGE("data len %d\n",data_length);
m_pcmbuf = new JBuf(data_length);
int rst = wav_read_data(fhnd,(unsigned char*)m_pcmbuf->data(),data_length);
wav_read_close(fhnd);
int wavsample = data_length/2;
m_pcmsample = wavsample + 2*MFCC_OFFSET;
int seca = m_pcmsample / MFCC_WAVCHUNK;
int secb = m_pcmsample % MFCC_WAVCHUNK;
if(secb>0){
//m_pcmsample = wavsample + 2*MFCC_OFFSET + MFCC_WAVCHUNK - secb;
//seca++;
}
int mellast = secb?(secb /160 +1):0;
int bnflast = secb?((mellast*0.25f)-0.75f):0;
int wavsize = seca*MFCC_WAVCHUNK + secb;
int melsize = seca*MFCC_MELBASE+mellast;
int bnfsize = seca*MFCC_BNFBASE+bnflast;
int calcsize = seca+1;
m_wavmat = new JMat(MFCC_WAVCHUNK,calcsize,1);
m_wavmat->zeros();
short* ps = (short*)m_pcmbuf->data();
float* pd = (float*)m_wavmat->data();
float* pf = pd+MFCC_OFFSET;
for(int k=0;k<wavsample;k++){
*pf++ = (float)(*ps++/ 32767.f);
}
m_melmat = new JMat(MFCC_MELCHUNK,MFCC_MELBASE*calcsize,1);
m_melmat->zeros();
//m_bnfmat = new JMat(MFCC_BNFCHUNK,MFCC_BNFBASE*calcsize,1);
//m_bnfmat->zeros();
//
//printf("===seca %d secb %d mellast %d\n",seca,secb,mellast);
//m_bnfmat = new JMat(
calcmfcc(m_wavmat,m_melmat);
float* mel = m_melmat->fdata();
for(int k=0;k<seca;k++){
float* bnf = bnfcache->secBuf(k)->fdata();
calcbnf(mel,MFCC_MELBASE,bnf,MFCC_BNFBASE);
//dumpfloat(bnf,10);
mel+=MFCC_MELBASE*MFCC_MELCHUNK;
//bnf+=MFCC_BNFBASE*MFCC_BNFCHUNK;
}
if(mellast){
//fix last
int inxsec = seca ;//seca?(seca+1):0;
printf("===indexsec %d\n",inxsec);
float* bnf = bnfcache->secBuf(inxsec)->fdata();
calcbnf(mel,mellast,bnf,bnflast);
//dumpfloat(bnf,10);
//calcbnf(mel,MFCC_MELBASE,bnf,MFCC_BNFBASE);
}
int* arr = bnfcache->tagarr();
//
arr[0] = wavsize;
arr[1] = m_pcmsample;
arr[2] = seca;
arr[3] = secb;
float secs = wavsample *1.0f/ MFCC_RATE;
int bnfblock = secs*MFCC_FPS;
if(bnfblock>(bnfsize-10))bnfblock = bnfsize-10;
arr[4] = melsize;
arr[5] = bnfsize;
arr[6] = bnfblock;
/*
for(int k=0;k<10;k++){
float* bnf = m_bnfmat->frow(k);
printf("==%d =bnf %f\n",k,*bnf);
}
*/
//*pmat = m_bnfmat;
delete m_pcmbuf;
delete m_wavmat;
delete m_melmat ;
return bnfblock;
}
float* Wenet::nextbnf(JMat* bnfmat,int index){
int* arr = bnfmat->tagarr();
int bnfsize = arr[5] ;
int bnfblock = arr[6] ;
LOGD("===index %d bnfsize %d bnfblock %d\n",index,bnfsize,bnfblock);
if(bnfblock>bnfsize)return NULL;
if(index>=bnfblock)return NULL;
float* buf = bnfmat->fdata();
buf += index*MFCC_BNFCHUNK+MFCC_BNFCHUNK;
return buf;
}
int Wenet::calcmfcc(float* fwav,float* mel2){
int rst = 0;
int melcnt = MFCC_WAVCHUNK/160+1;
rst = log_mel(fwav,MFCC_WAVCHUNK, 16000,mel2);
return rst;
}
int Wenet::calcmfcc(JMat* mwav,JMat* mmel){
int rst = 0;
int melcnt = MFCC_WAVCHUNK/160+1;
for(size_t k=0;k<mwav->height();k++){
float* fwav = mwav->frow(k);
float* mel2 = mmel->frow(k);
rst = log_mel(fwav,MFCC_WAVCHUNK, 16000,mel2);
}
return rst;
}
#ifdef WENET_MAIN
int main(int argc,char** argv){
Wenet net("../model","wenet");
net->nextwav("../mybin/a.wav");
return 0;
}
#endif

View File

@@ -0,0 +1,22 @@
#pragma once
#include "jmat.h"
#include "aimodel.h"
#include <vector>
#include "wavcache.h"
//#include <onnx/onnxruntime_cxx_api.h>
class Wenet{
private:
OnnxModel *m_model = nullptr;
void initModel(const char* modelfn);
public:
int calcmfcc(JMat* mwav,JMat* mmel);
int calcmfcc(float* fwav,float* mel2);
int calcbnf(float* melbin,int melnum,float* bnfbin,int bnfnum);
//int nextwav(const char* wavfile,JMat** pmat);
int nextwav(const char* wavfile,MBnfCache* bnfcache);
float* nextbnf(JMat* bnfmat,int index);
Wenet(const char* modeldir,const char* modelid);
Wenet(const char* modelfn);
~Wenet();
};

View File

@@ -0,0 +1,53 @@
#include <Log.h>
#include "AudioTrackJni.h"
#include "JniHelper.h"
#define TAG "AudioTrackJni"
AudioTrackJni::AudioTrackJni(void* obj):AudioTrack(obj) {
JNIEnv *env = JniHelper::getJNIEnv();
audioTrackObj = env->NewGlobalRef(obj);
audioTrackClass = env->GetObjectClass(obj);
audioTrackStart = env->GetMethodID(audioTrackClass, "open", "(IIII)V");
audioTrackWrite = env->GetMethodID(audioTrackClass, "write", "(Ljava/nio/ByteBuffer;I)I");
audioTrackStop = env->GetMethodID(audioTrackClass, "close", "()V");
}
AudioTrackJni::~AudioTrackJni() = default;
void AudioTrackJni::start(int sampleRate, int sampleFormat, int channels, int bytesPerSample) {
LOGI(TAG, "start %d %d %d %d", sampleRate, sampleFormat, channels, bytesPerSample);
bool attach = JniHelper::attachCurrentThread();
if (audioTrackObj != nullptr && audioTrackStart != nullptr) {
JniHelper::callVoidMethod(audioTrackObj, audioTrackStart, sampleRate, sampleFormat,
channels, bytesPerSample);
}
if (attach) {
JniHelper::detachCurrentThread();
}
}
int AudioTrackJni::write(uint8_t *buffer, int size) {
bool attach = JniHelper::attachCurrentThread();
if (audioTrackObj != nullptr && audioTrackWrite != nullptr) {
JNIEnv *env = JniHelper::getJNIEnv();
jobject byteBuffer = JniHelper::createByteBuffer(env, buffer, size);
int result = JniHelper::callIntMethod(audioTrackObj, audioTrackWrite, byteBuffer, size);
env->DeleteLocalRef(byteBuffer);
return result;
}
if (attach) {
JniHelper::detachCurrentThread();
}
return -1;
}
void AudioTrackJni::stop() {
bool attach = JniHelper::attachCurrentThread();
if (audioTrackObj != nullptr && audioTrackStop != nullptr) {
JniHelper::callVoidMethod(audioTrackObj, audioTrackStop);
}
if (attach) {
JniHelper::detachCurrentThread();
}
}

View File

@@ -0,0 +1,32 @@
#ifndef GPLAYER_AUDIOTRACKJNI_H
#define GPLAYER_AUDIOTRACKJNI_H
#include <jni.h>
#include "FrameSource.h"
#include "AudioTrack.h"
class AudioTrackJni:public AudioTrack {
public:
AudioTrackJni(jobject obj);
~AudioTrackJni();
virtual void start(int sampleRate, int sampleFormat, int channels, int bytesPerSample);
virtual int write(uint8_t *buffer, int size);
virtual void stop();
private:
jclass audioTrackClass;
jmethodID audioTrackStart;
jmethodID audioTrackWrite;
jmethodID audioTrackStop;
private:
jobject audioTrackObj;
};
#endif //GPLAYER_AUDIOTRACKJNI_H

View File

@@ -0,0 +1,273 @@
#include <android/asset_manager_jni.h>
#include <android/native_window_jni.h>
#include <android/native_window.h>
#include <android/log.h>
#include <jni.h>
#include <string>
#include <vector>
#include <unistd.h>
#include "GDigit.h"
#include "Log.h"
#include "MsgcbJni.h"
#include "JniHelper.h"
#include "aesmain.h"
#if __ARM_NEON
#include <arm_neon.h>
#endif // __ARM_NEON
//
//
static GDigit* g_digit = 0;
static JMat* g_gpgmat = NULL;
static MessageCb* g_msgcb = new MessageCb();
static int g_width = 540;
static int g_height = 960;
static int g_taskid = -1;
#define TAG "tooken"
extern "C" {
JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *reserved) {
LOGD(TAG, "JNI_OnLoad");
g_digit = new GDigit(g_width,g_height,g_msgcb);
JniHelper::sJavaVM = vm;
return JNI_VERSION_1_4;
}
JNIEXPORT void JNI_OnUnload(JavaVM *vm, void *reserved) {
LOGI(TAG, "unload");
if(g_digit){
delete g_digit;
g_digit = nullptr;
}
if(g_msgcb){
delete g_msgcb;
g_msgcb = nullptr;
}
}
static std::string getStringUTF(JNIEnv *env, jstring obj) {
char *c_str = (char *) env->GetStringUTFChars(obj, nullptr);
std::string tmpString = std::string(c_str);
env->ReleaseStringUTFChars(obj, c_str);
return tmpString;
}
JNIEXPORT jint JNICALL Java_com_btows_ncnntest_SCRFDNcnn_initModel(JNIEnv *env, jobject thiz, jstring cfgtxt){
return -1;
}
JNIEXPORT jint JNICALL Java_com_btows_ncnntest_SCRFDNcnn_createdigit(JNIEnv *env, jobject thiz, jint taskid,jobject msgcbobj){
LOGI(TAG, "create");
g_taskid = taskid;
return 0;
}
JNIEXPORT jint JNICALL Java_com_btows_ncnntest_SCRFDNcnn_releasedigit(JNIEnv *env, jobject thiz,jint taskid){
if(g_taskid==taskid){
g_digit->stop();
g_digit->recyle();
}
return 0;
}
JNIEXPORT jint JNICALL
Java_com_btows_ncnntest_SCRFDNcnn_config(JNIEnv *env, jobject thiz, jstring cfgtxt){
std::string str = getStringUTF(env,cfgtxt);
LOGI(TAG,"cfgstr %s",str.c_str());
g_digit->config(str.c_str());
LOGI(TAG,"cfgstr %s",str.c_str());
g_digit->prepare();
return 0;
}
JNIEXPORT jint JNICALL
Java_com_btows_ncnntest_SCRFDNcnn_start(JNIEnv *env, jobject thiz){
g_digit->start();
return 0;
}
JNIEXPORT jint JNICALL
Java_com_btows_ncnntest_SCRFDNcnn_stop(JNIEnv *env, jobject thiz){
g_digit->stop();
return 0;
}
JNIEXPORT jint JNICALL
Java_com_btows_ncnntest_SCRFDNcnn_netwav(JNIEnv *env, jobject thiz, jstring wavurl,jfloat duration){
std::string s_wav = getStringUTF(env,wavurl);
return g_digit->netwav(s_wav.c_str(),duration);
}
JNIEXPORT jint JNICALL
Java_com_btows_ncnntest_SCRFDNcnn_bgpic(JNIEnv *env, jobject thiz, jstring picfn){
std::string s_pic = getStringUTF(env,picfn);
return g_digit->bgpic(s_pic.c_str());
}
JNIEXPORT jint JNICALL
Java_com_btows_ncnntest_SCRFDNcnn_drawmskpic(JNIEnv *env, jobject thiz, jstring picfn,jstring mskfn){
std::string s_pic = getStringUTF(env,picfn);
std::string s_dump = getStringUTF(env,mskfn);
return g_digit->drawmskpic(s_pic.c_str(),s_dump.c_str());
}
JNIEXPORT jint JNICALL
Java_com_btows_ncnntest_SCRFDNcnn_drawmskbuf(JNIEnv *env, jobject thiz, jstring picfn,jstring mskfn,jbyteArray arrbuf,jbyteArray mskbuf,jint bufsize){
std::string s_pic = getStringUTF(env,picfn);
std::string s_dump = getStringUTF(env,mskfn);
jbyte *pixels = (jbyte *) env->GetPrimitiveArrayCritical(arrbuf, 0);
jbyte *pmsk = (jbyte *) env->GetPrimitiveArrayCritical(mskbuf, 0);
int size = bufsize;
int rst = g_digit->drawmskbuf(s_pic.c_str(),s_dump.c_str(),(char*)pixels,(char*)pmsk,size);
env->ReleasePrimitiveArrayCritical(arrbuf, pixels, 0);
env->ReleasePrimitiveArrayCritical(mskbuf, pmsk, 0);
return rst;
}
JNIEXPORT jint JNICALL Java_com_btows_ncnntest_SCRFDNcnn_mskrstpic(JNIEnv* env, jobject thiz,jstring picfile, jstring mskfile,jintArray arrbox, jint index,jstring fgfile){
std::string s_pic = getStringUTF(env,picfile);
std::string s_msk = getStringUTF(env,mskfile);
std::string s_fg = getStringUTF(env,fgfile);
jint *boxData = (jint*) env->GetIntArrayElements( arrbox, NULL);
int rst = g_digit->mskrstpic(index,s_pic.c_str(),(int*)boxData,s_msk.c_str(),s_fg.c_str());
env->ReleaseIntArrayElements( arrbox, boxData, 0);
return rst;
}
JNIEXPORT jint JNICALL
Java_com_btows_ncnntest_SCRFDNcnn_drawonebuf(JNIEnv *env, jobject thiz, jstring picfn,jbyteArray arrbuf,jint bufsize){
std::string s_pic = getStringUTF(env,picfn);
jbyte *pixels = (jbyte *) env->GetPrimitiveArrayCritical(arrbuf, 0);
int size = bufsize;
int rst = g_digit->drawonebuf(s_pic.c_str(),(char*)pixels,size);
env->ReleasePrimitiveArrayCritical(arrbuf, pixels, 0);
return rst;
}
JNIEXPORT jint JNICALL Java_com_btows_ncnntest_SCRFDNcnn_onerstbuf(JNIEnv* env, jobject thiz,jstring picfile, jintArray arrbox, jint index,jbyteArray arrbuf,jint bufsize){
std::string s_pic = getStringUTF(env,picfile);
jint *boxData = (jint*) env->GetIntArrayElements( arrbox, NULL);
jbyte *pixels = (jbyte *) env->GetPrimitiveArrayCritical(arrbuf, 0);
int size = bufsize;
int rst = g_digit->onerstbuf(index,s_pic.c_str(),(int*)boxData,(char*) pixels,size);
env->ReleasePrimitiveArrayCritical(arrbuf, pixels, 0);
env->ReleaseIntArrayElements( arrbox, boxData, 0);
return rst;
}
JNIEXPORT jint JNICALL Java_com_btows_ncnntest_SCRFDNcnn_mskrstbuf(JNIEnv* env, jobject thiz,jstring picfile, jstring mskfile,jintArray arrbox, jint index,jstring fgfile,jbyteArray arrbuf,jbyteArray mskbuf,jint bufsize){
std::string s_pic = getStringUTF(env,picfile);
std::string s_msk = getStringUTF(env,mskfile);
std::string s_fg = getStringUTF(env,fgfile);
jint *boxData = (jint*) env->GetIntArrayElements( arrbox, NULL);
jbyte *pixels = (jbyte *) env->GetPrimitiveArrayCritical(arrbuf, 0);
jbyte *pmsk = (jbyte *) env->GetPrimitiveArrayCritical(mskbuf, 0);
int size = bufsize;
int rst = g_digit->mskrstbuf(index,s_pic.c_str(),(int*)boxData,s_msk.c_str(),s_fg.c_str(),(char*) pixels,(char*)pmsk,size);
env->ReleasePrimitiveArrayCritical(arrbuf, pixels, 0);
env->ReleasePrimitiveArrayCritical(mskbuf, pmsk, 0);
env->ReleaseIntArrayElements( arrbox, boxData, 0);
return rst;
}
JNIEXPORT jint JNICALL
Java_com_btows_ncnntest_SCRFDNcnn_onewav(JNIEnv *env, jobject thiz, jstring wavfn,jstring dumpfn){
std::string s_wav = getStringUTF(env,wavfn);
std::string s_dump = getStringUTF(env,dumpfn);
return g_digit->newwav(s_wav.c_str(),s_dump.c_str());
}
JNIEXPORT jint JNICALL
Java_com_btows_ncnntest_SCRFDNcnn_onepic(JNIEnv *env, jobject thiz, jstring picfn,jstring dumpfn){
std::string s_pic = getStringUTF(env,picfn);
std::string s_dump = getStringUTF(env,dumpfn);
return g_digit->newpic(s_pic.c_str(),s_dump.c_str());
}
JNIEXPORT jint JNICALL
Java_com_btows_ncnntest_SCRFDNcnn_drawpic(JNIEnv *env, jobject thiz, jstring picfn){
std::string s_pic = getStringUTF(env,picfn);
return g_digit->drawpic(s_pic.c_str());
}
// public native boolean openCamera(int facing);
JNIEXPORT jint JNICALL Java_com_btows_ncnntest_SCRFDNcnn_onerst(JNIEnv* env, jobject thiz, jint index,jstring dumpfn) {
std::string s_dump = getStringUTF(env,dumpfn);
return g_digit->newrst(index,s_dump.c_str());
}
JNIEXPORT jint JNICALL Java_com_btows_ncnntest_SCRFDNcnn_netrst(JNIEnv* env, jobject thiz, jint index,jstring dumpfn) {
std::string s_dump = getStringUTF(env,dumpfn);
return g_digit->netrst(index,s_dump.c_str());
}
JNIEXPORT jint JNICALL Java_com_btows_ncnntest_SCRFDNcnn_picrst(JNIEnv* env, jobject thiz,jstring picfile, jintArray arrbox, jint index,jstring dumpfn){
std::string s_pic = getStringUTF(env,picfile);
std::string s_dump = getStringUTF(env,dumpfn);
jint *boxData = (jint*) env->GetIntArrayElements( arrbox, NULL);
int rst = g_digit->picrst(s_pic.c_str(),(int*)boxData,index,s_dump.c_str());
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "picrst %d",rst);
env->ReleaseIntArrayElements( arrbox, boxData, 0);
return rst;
}
JNIEXPORT jint JNICALL Java_com_btows_ncnntest_SCRFDNcnn_netrstpic(JNIEnv* env, jobject thiz,jstring picfile, jintArray arrbox, jint index,jstring dumpfn){
std::string s_pic = getStringUTF(env,picfile);
std::string s_dump = getStringUTF(env,dumpfn);
jint *boxData = (jint*) env->GetIntArrayElements( arrbox, NULL);
int rst = g_digit->netrstpic(s_pic.c_str(),(int*)boxData,index,s_dump.c_str());
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "picrst %d",rst);
env->ReleaseIntArrayElements( arrbox, boxData, 0);
return rst;
}
JNIEXPORT jint JNICALL Java_com_btows_ncnntest_SCRFDNcnn_reset(JNIEnv* env, jobject thiz){
//g_digit->reset_fps();
//g_digit->reset_inx();
g_digit->setSurface(NULL);
return 0;
}
// public native boolean setOutputWindow(Surface surface);
JNIEXPORT jint JNICALL Java_com_btows_ncnntest_SCRFDNcnn_setOutputWindow(JNIEnv* env, jobject thiz, jobject surface)
{
ANativeWindow* win = ANativeWindow_fromSurface(env, surface);
__android_log_print(ANDROID_LOG_DEBUG, "ncnn", "setOutputWindow %p", win);
g_digit->setSurface(win);
//g_digit->start();
return 0;
}
JNIEXPORT jint JNICALL
Java_com_btows_ncnntest_SCRFDNcnn_startgpg(JNIEnv *env, jobject thiz, jstring picfn,jstring gpgfn){
std::string s_pic = getStringUTF(env,picfn);
std::string s_gpg = getStringUTF(env,gpgfn);
if(!g_gpgmat)g_gpgmat = new JMat();
int rst = g_gpgmat->loadjpg(s_pic);
if(rst)return rst;
rst = g_gpgmat->savegpg(s_gpg);
return rst;
}
JNIEXPORT jint JNICALL
Java_com_btows_ncnntest_SCRFDNcnn_processmd5(JNIEnv *env, jobject thiz, jint kind,jstring infn,jstring outfn){
std::string s_in = getStringUTF(env,infn);
std::string s_out = getStringUTF(env,outfn);
int rst = mainenc(kind,(char*)s_in.c_str(),(char*)s_out.c_str());
return rst;
}
JNIEXPORT jint JNICALL
Java_com_btows_ncnntest_SCRFDNcnn_stopgpg(JNIEnv *env, jobject thiz){
if(g_gpgmat){
delete g_gpgmat;
g_gpgmat = NULL;
}
return 0;
}
}

View File

@@ -0,0 +1,384 @@
#include <malloc.h>
#include "JniHelper.h"
#include "Log.h"
#define TAG "JniHelper"
using namespace std;
JavaVM *JniHelper::sJavaVM = nullptr;
JNIEnv *JniHelper::getJNIEnv() {
if (sJavaVM == nullptr) {
LOGE(TAG, "sJavaVM is nullptr");
return nullptr;
}
JNIEnv *env = nullptr;
bool attached = false;
switch (sJavaVM->GetEnv((void **) &env, JNI_VERSION_1_4)) {
case JNI_OK:
break;
case JNI_EDETACHED:
if (sJavaVM->AttachCurrentThread(&env, nullptr) != 0) {
LOGE(TAG, "Could not attach current thread");
}
attached = true;
break;
case JNI_EVERSION:
LOGE(TAG, "Invalid java version");
break;
default:
break;
}
if (attached) {
sJavaVM->DetachCurrentThread();
}
return env;
}
bool JniHelper::attachCurrentThread() {
if (sJavaVM == nullptr) {
LOGE(TAG, "sJavaVM is nullptr");
return false;
}
JNIEnv *env = nullptr;
bool attached = false;
switch (sJavaVM->GetEnv((void **) &env, JNI_VERSION_1_4)) {
case JNI_OK:
break;
case JNI_EDETACHED:
if (sJavaVM->AttachCurrentThread(&env, nullptr) != 0) {
LOGE(TAG, "Could not attach current thread");
} else {
attached = true;
}
break;
case JNI_EVERSION:
LOGE(TAG, "Invalid java version");
break;
default:
break;
}
return attached;
}
void JniHelper::detachCurrentThread() {
sJavaVM->DetachCurrentThread();
}
void JniHelper::throwException(JNIEnv *env, const char *className, const char *msg) {
jclass exception = env->FindClass(className);
env->ThrowNew(exception, msg);
}
jstring JniHelper::newStringUTF(JNIEnv *env, const char *data) {
if (!data) return nullptr;
jstring str = nullptr;
int size = strlen(data);
jbyteArray array = env->NewByteArray(size);
if (!array) { // OutOfMemoryError exception has already been thrown.
LOGE(TAG, "convertString: OutOfMemoryError is thrown.");
} else {
env->SetByteArrayRegion(array, 0, size, (jbyte *) data);
jclass string_Clazz = env->FindClass("java/lang/String");
jmethodID string_initMethodID = env->GetMethodID(string_Clazz, "<init>",
"([BLjava/lang/String;)V");
jstring utf = env->NewStringUTF("UTF-8");
str = (jstring) env->NewObject(string_Clazz, string_initMethodID, array, utf);
env->DeleteLocalRef(utf);
env->DeleteLocalRef(array);
}
return str;
};
jobject JniHelper::createByteBuffer(JNIEnv *env, unsigned char *buffer, int size) {
if (env == nullptr || buffer == nullptr) {
return nullptr;
}
jobject byteBuffer = env->NewDirectByteBuffer(buffer, size);
//byteBuffer = env->NewGlobalRef(byteBuffer);
return byteBuffer;
}
jobject JniHelper::createByteBuffer(JNIEnv *env, int size) {
if (env == nullptr) {
return nullptr;
}
auto buffer = static_cast<uint8_t *>(malloc(static_cast<size_t>(size)));
jobject byteBuffer = env->NewDirectByteBuffer(buffer, size);
free(buffer);
return byteBuffer;
}
void JniHelper::deleteLocalRef(jobject jobj) {
JNIEnv *env = JniHelper::getJNIEnv();
if (env == nullptr || jobj == nullptr) {
return;
}
env->DeleteLocalRef(jobj);
}
string JniHelper::getStringUTF(JNIEnv *env, jstring obj) {
char *c_str = (char *) env->GetStringUTFChars(obj, nullptr);
string tmpString = std::string(c_str);
env->ReleaseStringUTFChars(obj, c_str);
return tmpString;
}
char *JniHelper::getCharArrayUTF(JNIEnv *env, jstring obj) {
char *c_str = (char *) env->GetStringUTFChars(obj, nullptr);
env->ReleaseStringUTFChars(obj, c_str);
return c_str;
}
void JniHelper::callVoidMethod(jobject obj, jmethodID methodId) {
if (sJavaVM == nullptr) {
LOGE(TAG, "sJavaVM is nullptr");
return;
}
JNIEnv *env = nullptr;
bool attached = false;
switch (sJavaVM->GetEnv((void **) &env, JNI_VERSION_1_4)) {
case JNI_OK:
break;
case JNI_EDETACHED:
if (sJavaVM->AttachCurrentThread(&env, nullptr) != 0) {
LOGE(TAG, "Could not attach current thread");
}
attached = true;
break;
case JNI_EVERSION:
LOGE(TAG, "Invalid java version");
break;
default:
break;
}
if (env != nullptr) {
env->CallVoidMethod(obj, methodId);
}
if (attached) {
sJavaVM->DetachCurrentThread();
}
}
void JniHelper::callVoidMethod(jobject obj, jmethodID methodId, jint arg1, jint arg2, jint arg3, jint arg4) {
if (sJavaVM == nullptr) {
LOGE(TAG, "sJavaVM is nullptr");
return;
}
JNIEnv *env = nullptr;
bool attached = false;
switch (sJavaVM->GetEnv((void **) &env, JNI_VERSION_1_4)) {
case JNI_OK:
break;
case JNI_EDETACHED:
if (sJavaVM->AttachCurrentThread(&env, nullptr) != 0) {
LOGE(TAG, "Could not attach current thread");
}
attached = true;
break;
case JNI_EVERSION:
LOGE(TAG, "Invalid java version");
break;
default:
break;
}
if (env != nullptr) {
env->CallVoidMethod(obj, methodId, arg1, arg2, arg3, arg4);
}
if (attached) {
sJavaVM->DetachCurrentThread();
}
}
void
JniHelper::callVoidMethod(jobject obj, jmethodID methodId, jint arg1, jint arg2, jint arg3,
jstring arg4, jstring arg5, jobject arg6) {
if (sJavaVM == nullptr) {
LOGE(TAG, "sJavaVM is nullptr");
return;
}
JNIEnv *env = nullptr;
bool attached = false;
switch (sJavaVM->GetEnv((void **) &env, JNI_VERSION_1_4)) {
case JNI_OK:
break;
case JNI_EDETACHED:
if (sJavaVM->AttachCurrentThread(&env, nullptr) != 0) {
LOGE(TAG, "Could not attach current thread");
}
attached = true;
break;
case JNI_EVERSION:
LOGE(TAG, "Invalid java version");
break;
default:
break;
}
if (env != nullptr) {
env->CallVoidMethod(obj, methodId, arg1, arg2, arg3, arg4, arg5, arg6);
}
if (attached) {
sJavaVM->DetachCurrentThread();
}
}
int JniHelper::callIntMethod(jobject obj, jmethodID methodId, jobject arg1, jint arg2) {
if (sJavaVM == nullptr) {
LOGE(TAG, "sJavaVM is nullptr");
return -1;
}
JNIEnv *env = nullptr;
bool attached = false;
switch (sJavaVM->GetEnv((void **) &env, JNI_VERSION_1_4)) {
case JNI_OK:
break;
case JNI_EDETACHED:
if (sJavaVM->AttachCurrentThread(&env, nullptr) != 0) {
LOGE(TAG, "Could not attach current thread");
}
attached = true;
break;
case JNI_EVERSION:
LOGE(TAG, "Invalid java version");
break;
default:
break;
}
int ret = -1;
if (env != nullptr) {
ret = env->CallIntMethod(obj, methodId, arg1, arg2);
}
if (attached) {
sJavaVM->DetachCurrentThread();
}
return ret;
}
void JniHelper::callStaticVoidMethod(jclass cls, jmethodID methodId, jint arg1) {
if (sJavaVM == nullptr) {
LOGE(TAG, "sJavaVM is nullptr");
return;
}
JNIEnv *env = nullptr;
bool attached = false;
switch (sJavaVM->GetEnv((void **) &env, JNI_VERSION_1_4)) {
case JNI_OK:
break;
case JNI_EDETACHED:
if (sJavaVM->AttachCurrentThread(&env, nullptr) != 0) {
LOGE(TAG, "Could not attach current thread");
}
attached = true;
break;
case JNI_EVERSION:
LOGE(TAG, "Invalid java version");
break;
default:
break;
}
if (env != nullptr) {
env->CallStaticVoidMethod(cls, methodId, arg1);
}
if (attached) {
sJavaVM->DetachCurrentThread();
}
}
jobject JniHelper::callObjectMethod(jobject obj, jmethodID methodId) {
if (sJavaVM == nullptr) {
LOGE(TAG, "sJavaVM is nullptr");
return nullptr;
}
JNIEnv *env = nullptr;
bool attached = false;
switch (sJavaVM->GetEnv((void **) &env, JNI_VERSION_1_4)) {
case JNI_OK:
break;
case JNI_EDETACHED:
if (sJavaVM->AttachCurrentThread(&env, nullptr) != 0) {
LOGE(TAG, "Could not attach current thread");
}
attached = true;
break;
case JNI_EVERSION:
LOGE(TAG, "Invalid java version");
break;
default:
break;
}
jobject ret = nullptr;
if (env != nullptr) {
ret = env->CallObjectMethod(obj, methodId);
}
if (attached) {
sJavaVM->DetachCurrentThread();
}
return ret;
}
jboolean JniHelper::callBooleanMethod(jobject obj, jmethodID methodId) {
if (sJavaVM == nullptr) {
LOGE(TAG, "sJavaVM is nullptr");
return false;
}
JNIEnv *env = nullptr;
bool attached = false;
switch (sJavaVM->GetEnv((void **) &env, JNI_VERSION_1_4)) {
case JNI_OK:
break;
case JNI_EDETACHED:
if (sJavaVM->AttachCurrentThread(&env, nullptr) != 0) {
LOGE(TAG, "Could not attach current thread");
}
attached = true;
break;
case JNI_EVERSION:
LOGE(TAG, "Invalid java version");
break;
default:
break;
}
jboolean ret;
if (env != nullptr) {
ret = env->CallBooleanMethod(obj, methodId);
}
if (attached) {
sJavaVM->DetachCurrentThread();
}
return ret;
}

View File

@@ -0,0 +1,50 @@
#ifndef GPLAYER_JNIHELPER_H
#define GPLAYER_JNIHELPER_H
#include <jni.h>
#include <string>
using namespace std;
class JniHelper {
public:
static JNIEnv *getJNIEnv();
static bool attachCurrentThread();
static void detachCurrentThread();
static void throwException(JNIEnv *env, const char *className, const char *msg);
static jstring newStringUTF(JNIEnv *env, const char *data);
static string getStringUTF(JNIEnv *env, jstring obj);
static char *getCharArrayUTF(JNIEnv *env, jstring obj);
static jobject createByteBuffer(JNIEnv *env, unsigned char *buffer, int size);
static jobject createByteBuffer(JNIEnv *env, int size);
static void deleteLocalRef(jobject jobj);
static void callVoidMethod(jobject obj, jmethodID methodId);
static void callVoidMethod(jobject obj, jmethodID methodId, jint arg1, jint arg2, jint arg3, jint arg4);
static void callVoidMethod(jobject obj, jmethodID methodId, jint arg1, jint arg2,
jint arg3, jstring arg4, jstring arg5, jobject arg6);
static int callIntMethod(jobject obj, jmethodID methodId, jobject arg1, jint arg2);
static void callStaticVoidMethod(jclass cls, jmethodID methodId, jint arg1);
static jobject callObjectMethod(jobject obj, jmethodID methodId);
static jboolean callBooleanMethod(jobject obj, jmethodID methodId);
public:
static JavaVM *sJavaVM;
};
#endif //GPLAYER_JNIHELPER_H

View File

@@ -0,0 +1,56 @@
#include <Log.h>
#include <JniHelper.h>
#include "MsgcbJni.h"
#define TAG "MediaSourceJni"
MsgcbJni::MsgcbJni(jobject obj) {
LOGI(TAG, "create MsgcbJni");
JNIEnv *env = JniHelper::getJNIEnv();
msgcbJObj = env->NewGlobalRef(obj);
jclass sourceClass = env->GetObjectClass(obj);
onMessageCallbackMethod = env->GetMethodID(sourceClass, "onMessageCallback",
"(IIJLjava/lang/String;Ljava/lang/String;Ljava/lang/Object;)V");
}
MsgcbJni::~MsgcbJni() {
bool attach = JniHelper::attachCurrentThread();
JNIEnv *env = JniHelper::getJNIEnv();
if (msgcbJObj != nullptr && env) {
env->DeleteGlobalRef(msgcbJObj);
msgcbJObj = nullptr;
onMessageCallbackMethod = nullptr;
}
if (attach) {
JniHelper::detachCurrentThread();
}
LOGE(TAG, "MsgcbJni destroyed");
}
void
MsgcbJni::onMessageCallback(int msgId, int arg1, long arg2, const char *msg1, const char *msg2) {
onMessageCallback(msgId, arg1, arg2, msg1, msg2, (jobject)nullptr);
}
void
MsgcbJni::onMessageCallback(int msgId, int arg1, long arg2, const char *msg1, const char *msg2, jobject obj) {
if (msgcbJObj != nullptr && onMessageCallbackMethod != nullptr) {
bool attach = JniHelper::attachCurrentThread();
JNIEnv *env = JniHelper::getJNIEnv();
jstring jMsg1 = msg1 ? JniHelper::newStringUTF(env, msg1) : nullptr;
jstring jMsg2 = msg2 ? JniHelper::newStringUTF(env, msg2) : nullptr;
JniHelper::callVoidMethod(msgcbJObj, onMessageCallbackMethod, msgId, arg1, arg2, jMsg1,
jMsg2, obj);
if (jMsg1) {
env->DeleteLocalRef(jMsg1);
}
if (jMsg2) {
env->DeleteLocalRef(jMsg2);
}
if (attach) {
JniHelper::detachCurrentThread();
}
}
}

View File

@@ -0,0 +1,25 @@
#ifndef GPLAYER_MSGCBERJNI_H
#define GPLAYER_MSGCBERJNI_H
#include <jni.h>
#include <string>
#include "CommObj.h"
class MsgcbJni:public MessageCb {
public:
MsgcbJni(jobject obj);
virtual ~MsgcbJni();
virtual void onMessageCallback(int msgId, int arg1, long arg2, const char *msg1, const char *msg2);
virtual void onMessageCallback(int msgId, int arg1, long arg2, const char *msg1, const char *msg2, jobject obj);
private:
jobject msgcbJObj;
jmethodID onMessageCallbackMethod;
};
#endif //GPLAYER_GPLAYERJNI_H

View File

@@ -0,0 +1,454 @@
#include "kmat.h"
#define __GARM__
#ifdef __GARM__
#include "ppl/cv/arm/resize.h"
#include "ppl/cv/arm/convertto.h"
#include "ppl/cv/arm/split.h"
#include "ppl/cv/arm/merge.h"
#include "ppl/cv/arm/convertto.h"
int KMat::vtacc(KMat* mat){
float* pt0 = (float*)m_buf;
const int stride = 532*532;
uint8_t pct[stride];
float* p0 = pt0;
float* p1 = p0+stride;
float* p2 = p1+stride;
float* psum = mat->fdata();
float* pd = psum;
for(int k=0;k<stride;k++){
*pd++ = *p0+++*p1+++*p2++;
}
float* pmean = psum + stride;
float* pt = pmean;
pd = psum;
p1 = pt0+stride;
for(int k=0;k<stride;k++){
*pt++ = (*pd++-*p1++)*0.5f;
}
uint8_t* pc = pct;
p1 = pt0+stride;
pt = pmean;
for(int k=0;k<stride;k++){
*pc++ = *p1++>*pt++;
}
memcpy(psum,pt0,stride*sizeof(float));
memcpy(psum+2*stride,pt0+2*stride,stride*sizeof(float));
//float* ped = pt0+stride;
pc = pct;
p1 = pt0+stride;
pt = pmean;
//pd = ped;
for(int k=0;k<stride;k++){
if(*pc++){
pt++;p1++;
}else{
*pt++ = *p1++;
}
//*pd++ = *pc?*pt++:*p1++;
}
return 0;
}
int KMat::resize(KMat* mat){
if(m_dev!=mat->m_dev)return -1;
ppl::cv::arm::ResizeLinear<float, 3>(m_height,m_width,m_stride,(float*)m_buf, mat->m_height, mat->m_width, mat->m_stride, (float*)mat->m_buf);
return 0;
}
int KMat::img2hwc(KMat* mat){
if(m_dev!=mat->m_dev)return -1;
if(m_channel!=3)return -2;
float* gpu_split = (float*)mat->m_buf;
float* gpu_s0 = gpu_split ;
int line = mat->m_stride*mat->m_height;
float* gpu_s1 = gpu_split + line;
float* gpu_s2 = gpu_split + line*2;
ppl::cv::arm::Split3Channels<float>(m_height, m_width,m_stride, (float*)m_buf, mat->m_width,gpu_s2,gpu_s1,gpu_s0);
return 0;
}
int KMat::hwc2img(KMat* mat){
if(m_dev!=mat->m_dev)return -1;
if(m_channel!=3)return -2;
return 0;
}
int KMat::downzero(KMat* mat){
if(m_dev!=mat->m_dev)return -1;
if((m_height!=mat->m_height)||(m_width!=mat->m_width))return -2;
if(!mat){
float* dst = (float*)m_buf;
for(int k=0;k< m_height;k++){
float* line = dst;
for(int m=0;m<m_width*m_channel;m++){
*line = *line *2.0f-1.0f;
line++;
}
dst+=m_stride;
}
}else{
float* src = (float*)m_buf;
float* dst = (float*)mat->m_buf;
for(int k=0;k< m_height;k++){
float* ls = src;
float* ld = dst;
for(int m=0;m<m_width*m_channel;m++){
*ld++ = *ls++ *2.0f-1.0f;
}
src+= m_stride;
dst+= mat->m_stride;
}
}
return 0;
}
int KMat::upzero(KMat* mat ){
if(m_dev!=mat->m_dev)return -1;
if((m_height!=mat->m_height)||(m_width!=mat->m_width))return -2;
if(!mat){
float* dst = (float*)m_buf;
for(int k=0;k< m_height;k++){
float* line = dst;
for(int m=0;m<m_width*m_channel;m++){
*line = (*line+1.0f) /2.0f;
line++;
}
dst+=m_stride;
}
}else{
float* src = (float*)m_buf;
float* dst = (float*)mat->m_buf;
for(int k=0;k< m_height;k++){
float* ls = src;
float* ld = dst;
for(int m=0;m<m_width*m_channel;m++){
*ld++ = (*ls++ +1.0f) /2.0f;
}
src+= m_stride;
dst+= mat->m_stride;
}
}
return 0;
}
int KMat::filtermask(KMat* msk, KMat* mat){
if(!msk)return -1;
if((msk->width()!=m_width) ||(msk->height()!=m_height))return -2;
if(mat==nullptr){
float* dst = (float*)m_buf;
float* mk = (float*)msk->m_buf;
if(msk->m_channel==3) {
for (int k = 0; k < m_height; k++) {
float *line = dst;
float *mm = mk;
for (int m = 0; m < m_width; m++) {
*line = *line * (*mm);
line++;
mm++;
}
dst += m_stride;
mk += msk->m_stride;
}
}else {
for (int k = 0; k < m_height; k++) {
float *line = dst;
float *mm = mk;
for (int m = 0; m < m_width; m++) {
for(int n=0;n<m_channel;n++){
*line = *line * (*mm);
line++;
}
mm++;
}
dst += m_stride;
mk += msk->m_stride;
}
}
}else{
if(mat->width()!=msk->width()||mat->height()!=msk->height()) return -3;
float* src = (float*)m_buf;
float* dst = (float*)mat->m_buf;
float* mk = (float*)msk->m_buf;
if(msk->m_channel==3) {
for (int k = 0; k < m_height; k++) {
float *line = dst;
float *mm = mk;
float *ms = src;
for (int m = 0; m < m_width; m++) {
*line++ = *ms++ * (*mm++);
}
dst += mat->m_stride;
src += m_stride;
mk += msk->m_stride;
}
}else{
for (int k = 0; k < m_height; k++) {
float *line = dst;
float *mm = mk;
float *ms = src;
for (int m = 0; m < m_width; m++) {
for(int n=0;n<m_channel;n++){
*line++ = *ms++ * (*mm);
}
mm++;
}
dst += mat->m_stride;
src += m_stride;
mk += msk->m_stride;
}
}
}
return 0;
}
int KMat::clamp(float minval,float maxval,KMat* mat){
if(!mat){
float* dst = (float*)m_buf;
for(int k=0;k< m_height;k++){
float* line = dst;
for(int m=0;m<m_width*m_channel;m++){
if(*line<minval){
*line = minval;
}else if(*line>maxval){
*line = maxval;
}
line++;
}
dst+=m_stride;
}
}else{
float* src = (float*)m_buf;
float* dst = (float*)mat->m_buf;
for(int k=0;k< m_height;k++){
float* ls = src;
float* ld = dst;
for(int m=0;m<m_width*m_channel;m++){
if(*ls<minval){
*ls = minval;
}else if(*ls>maxval){
*ls = maxval;
}else{
*ld = *ls;
}
*ld++ = *ls++;
}
src+= m_stride;
dst+= mat->m_stride;
}
}
return 0;
}
int KMat::blend(KMat* msk,KMat* bgmat,KMat* dstmat){
if(!msk)return -1;
if((msk->width()!=m_width) ||(msk->height()!=m_height))return -2;
if(dstmat==nullptr){
float* dst = (float*)m_buf;
float* mk = (float*)msk->m_buf;
float* bg = (float*)bgmat->m_buf;
if(msk->m_channel==3) {
for (int k = 0; k < m_height; k++) {
float *dd = dst;
float *mm = mk;
float *bb = bg;
for (int m = 0; m < m_width; m++) {
float ratio = *mm;
*dd = *dd * ratio + *bb*(1-ratio);
dd++; mm++; bb++;
}
dst += m_stride;
mk += msk->m_stride;
bg += bgmat->m_stride;
}
}else {
for (int k = 0; k < m_height; k++) {
float *dd = dst;
float *mm = mk;
float *bb = bg;
for (int m = 0; m < m_width; m++) {
float ratio = *mm;
for(int n=0;n<m_channel;n++){
*dd = *dd * ratio + *bb*(1-ratio);
dd++;
bb++;
}
mm++;
}
dst += m_stride;
mk += msk->m_stride;
bg += bgmat->m_stride;
}
}
}else{
if(dstmat->width()!=msk->width()||dstmat->height()!=msk->height()) return -3;
float* src = (float*)m_buf;
float* dst = (float*)dstmat->m_buf;
float* mk = (float*)msk->m_buf;
float* bg = (float*)bgmat->m_buf;
if(msk->m_channel==3) {
for (int k = 0; k < m_height; k++) {
float *dd = dst;
float *mm = mk;
//float *ss = src;
float *bb = bg;
for (int m = 0; m < m_width; m++) {
float ratio = *mm;
*dd = *dd * ratio + *bb*(1-ratio);
dd++; mm++; bb++;
}
dst += dstmat->m_stride;
src += m_stride;
mk += msk->m_stride;
bg += bgmat->m_stride;
}
}else{
for (int k = 0; k < m_height; k++) {
float *dd = dst;
float *mm = mk;
//float *ss = src;
float *bb = bg;
for (int m = 0; m < m_width; m++) {
float ratio = *mm;
for(int n=0;n<m_channel;n++){
*dd = *dd * ratio + *bb*(1-ratio);
src += m_stride;
dd++; bb++;
}
mm++;
}
dst += dstmat->m_stride;
src += m_stride;
mk += msk->m_stride;
bg += bgmat->m_stride;
}
}
}
return 0;
}
int KMat::wherezero(KMat* msk, KMat* bgmat, KMat* dstmat){
if(!msk)return -1;
if((msk->width()!=m_width) ||(msk->height()!=m_height))return -2;
if(dstmat==nullptr){
float* dst = (float*)m_buf;
float* mk = (float*)msk->m_buf;
float* bg = (float*)bgmat->m_buf;
if(msk->m_channel==3) {
for (int k = 0; k < m_height; k++) {
float *dd = dst;
float *mm = mk;
float *bb = bg;
for (int m = 0; m < m_width; m++) {
float ratio = *mm;
*dd = ratio==0?*dd:*bb;
dd++; mm++; bb++;
}
dst += m_stride;
mk += msk->m_stride;
bg += bgmat->m_stride;
}
}else {
for (int k = 0; k < m_height; k++) {
float *dd = dst;
float *mm = mk;
float *bb = bg;
for (int m = 0; m < m_width; m++) {
float ratio = *mm;
for(int n=0;n<m_channel;n++){
*dd = ratio==0?*dd:*bb;
dd++;
bb++;
}
mm++;
}
dst += m_stride;
mk += msk->m_stride;
bg += bgmat->m_stride;
}
}
}else{
if(dstmat->width()!=msk->width()||dstmat->height()!=msk->height()) return -3;
float* src = (float*)m_buf;
float* dst = (float*)dstmat->m_buf;
float* mk = (float*)msk->m_buf;
float* bg = (float*)bgmat->m_buf;
if(msk->m_channel==3) {
for (int k = 0; k < m_height; k++) {
float *dd = dst;
float *mm = mk;
//float *ss = src;
float *bb = bg;
for (int m = 0; m < m_width; m++) {
float ratio = *mm;
*dd = ratio==0?*dd:*bb;
dd++; mm++; bb++;
}
dst += dstmat->m_stride;
src += m_stride;
mk += msk->m_stride;
bg += bgmat->m_stride;
}
}else{
for (int k = 0; k < m_height; k++) {
float *dd = dst;
float *mm = mk;
//float *ss = src;
float *bb = bg;
for (int m = 0; m < m_width; m++) {
float ratio = *mm;
for(int n=0;n<m_channel;n++){
*dd = ratio==0?*dd:*bb;
src += m_stride;
dd++; bb++;
}
mm++;
}
dst += dstmat->m_stride;
src += m_stride;
mk += msk->m_stride;
bg += bgmat->m_stride;
}
}
}
return 0;
}
int KMat::cvtfloat(KMat* dst,float scale,float delta){
ppl::cv::arm::ConvertTo<uint8_t,float,3>(m_height,m_width,m_stride,(uint8_t*)m_buf
,m_width*m_channel,(float*)m_buf,scale,delta);
return 0;
}
int KMat::cvtuint8(KMat* dst,float scale,float delta){
ppl::cv::arm::ConvertTo<float,uint8_t,3>(m_height,m_width,m_stride,(float*)m_buf ,m_width*m_channel,(uint8_t*)m_buf,scale,delta);
return 0;
}
void KMat::initgh(){
m_dev = 0;
}
KMat::KMat(int w,int h,float *buf ,int c ,int d ):JMat(w,h,buf,c,d){
//
}
KMat::KMat(int w,int h,uint8_t *buf ,int c ,int d ):JMat(w,h,buf,c,d){
//
}
KMat::KMat(int w,int h,int c ,int d ,int b):JMat(w,h,c,d,b){
}
KMat::~KMat(){
//
}
#endif

View File

@@ -0,0 +1,105 @@
#include "ndkvid.h"
void MyVideo::reset(){
renderstart = -1;
sawInputEOS = false;
sawOutputEOS = false;
isPlaying = false;
renderonce = true;
}
MyVideo::MyVideo(const char* videofile){
int len = strlen(videofile);
memcpy(filename,videofile,len+1);
reset();
}
MyVideo::~MyVideo(){
}
int MyVideo::open(){
ex = AMediaExtractor_new();
media_status_t err = AMediaExtractor_setDataSource( ex, filename);
if (err != AMEDIA_OK) return -2;
int numtracks = AMediaExtractor_getTrackCount(ex);
for (int i = 0; i < numtracks; i++) {
AMediaFormat *format = AMediaExtractor_getTrackFormat(ex, i);
const char *s = AMediaFormat_toString(format);
const char *mime;
if (!AMediaFormat_getString(format, AMEDIAFORMAT_KEY_MIME, &mime)) {
return -3;
} else if (!strncmp(mime, "video/", 6)) {
AMediaExtractor_selectTrack(ex, i);
AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_FRAME_RATE, &frameRate) ;
AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_WIDTH, &frameWidth) ;
AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_HEIGHT, &frameHeight) ;
int64_t dur = 0;
AMediaFormat_getInt64(format, AMEDIAFORMAT_KEY_DURATION, &dur) ;
duration = dur;
codec = AMediaCodec_createDecoderByType(mime);
isPlaying = true;
//AMediaCodec_configure(codec, format, texture->getRenderWindow(), NULL, 0);
AMediaCodec_start(codec);
}
AMediaFormat_delete(format);
}
return 0;
}
int MyVideo::close(){
AMediaCodec_stop(codec);
AMediaCodec_delete(codec);
AMediaExtractor_delete(ex);
sawInputEOS = true;
sawOutputEOS = true;
return 0;
}
int MyVideo::next(JMat* mat){
size_t outsize = 0;
ssize_t bufidx = -1;
int64_t presentationTimeUs = -1;
while(!sawInputEOS || !sawOutputEOS){
if(!sawInputEOS){
bufidx = AMediaCodec_dequeueInputBuffer(codec, 0);
if (bufidx >= 0) {
size_t bufsize;
auto buf = AMediaCodec_getInputBuffer(codec, bufidx, &bufsize);
auto sampleSize = AMediaExtractor_readSampleData(ex, buf, bufsize);
if (sampleSize < 0) {
sampleSize = 0;
sawInputEOS = true;
isPlaying = false;
}
presentationTimeUs = AMediaExtractor_getSampleTime(ex);
AMediaCodec_queueInputBuffer( codec, bufidx, 0, sampleSize, presentationTimeUs, sawInputEOS ? AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM : 0);
AMediaExtractor_advance(ex);
}
}
if (!sawOutputEOS) {
AMediaCodecBufferInfo info;
bufidx = AMediaCodec_dequeueOutputBuffer(codec, &info, 0);
if (bufidx >= 0) {
if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) {
sawOutputEOS = true;
isPlaying = false;
}
position = presentationTimeUs;
if(mat){
uint8_t* output = AMediaCodec_getOutputBuffer(codec, bufidx, &outsize);
if(outsize>0){
memcpy(mat->data(),output,outsize);
}
}
//
AMediaCodec_releaseOutputBuffer(codec, bufidx, false);
break;
} else if (bufidx == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) {
auto format = AMediaCodec_getOutputFormat(codec);
AMediaFormat_delete(format);
}
}
}
return outsize;
}

View File

@@ -0,0 +1,49 @@
#pragma once
#include <android/log.h>
#include <android/native_window.h>
#include "jmat.h"
#include <android/native_window_jni.h>
#include <android/surface_texture.h>
#include <media/NdkMediaCodec.h>
#include <media/NdkMediaExtractor.h>
#define FRAME_DIST 40000
class MyVideo {
private:
char filename[1024];
int32_t frameHeight = 0;
int32_t frameWidth = 0;
int32_t frameRate = 0;
int32_t duration = 0;
int32_t position = 0;
int64_t renderstart;
bool sawInputEOS = false;
bool sawOutputEOS = false;
bool isPlaying = false;
bool renderonce = true;
AMediaExtractor *ex = nullptr;
AMediaCodec *codec = nullptr;
void indexsample(long time);
void reset();
int pre();
public:
int32_t getFrameHeight() const{return frameHeight;};
int32_t getFrameWidth() const{return frameWidth;};
int32_t getDuration() const{return duration;};
int32_t getPosition() const{return position;};
bool isEof(){return sawOutputEOS||sawInputEOS ;};
bool getPlaying(){return isPlaying ;};
int open();
int close();
int next(JMat* mat);
MyVideo(const char* videofile);
~MyVideo();
};

View File

@@ -0,0 +1,412 @@
#include "ndkwin.h"
MyDisp::MyDisp(){
sensor_manager = 0;
sensor_event_queue = 0;
accelerometer_sensor = 0;
win = 0;
accelerometer_orientation = 0;
sensor_manager = ASensorManager_getInstance();
accelerometer_sensor = ASensorManager_getDefaultSensor(sensor_manager, ASENSOR_TYPE_ACCELEROMETER);
}
MyDisp::~MyDisp(){
if (accelerometer_sensor)
{
ASensorEventQueue_disableSensor(sensor_event_queue, accelerometer_sensor);
accelerometer_sensor = 0;
}
if (sensor_event_queue)
{
ASensorManager_destroyEventQueue(sensor_manager, sensor_event_queue);
sensor_event_queue = 0;
}
if (win) {
ANativeWindow_release(win);
}
}
int MyDisp::draw_text(cv::Mat& rgb,const char* text)
{
//const char text[] = "unsupported";
int baseLine = 0;
cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 1.0, 1, &baseLine);
int y = (rgb.rows - label_size.height) / 10;
int x = (rgb.cols - label_size.width) / 10;
cv::rectangle(rgb, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)),
cv::Scalar(255, 255, 255), -1);
cv::putText(rgb, text, cv::Point(x, y + label_size.height),
cv::FONT_HERSHEY_SIMPLEX, 1.0, cv::Scalar(0, 0, 0));
return 0;
}
static double t0 = 0.f;
static float fps_history[10] = {0.f};
int MyDisp::reset_fps(){
t0 = 0;
memset(fps_history,0,10*sizeof(float));
return 0;
}
int MyDisp::draw_fps(cv::Mat& rgb)
{
// resolve moving average
float avg_fps = 0.f;
{
double t1 = ncnn::get_current_time();
if (t0 == 0.f)
{
t0 = t1;
return 0;
}
float fps = 1000.f / (t1 - t0);
t0 = t1;
for (int i = 9; i >= 1; i--)
{
fps_history[i] = fps_history[i - 1];
}
fps_history[0] = fps;
if (fps_history[9] == 0.f)
{
return 0;
}
for (int i = 0; i < 10; i++)
{
avg_fps += fps_history[i];
}
avg_fps /= 10.f;
}
char text[32];
sprintf(text, "FPS=%.2f", avg_fps);
int baseLine = 0;
cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
int y = 0;
int x = rgb.cols - label_size.width;
cv::rectangle(rgb, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)),
cv::Scalar(255, 255, 255), -1);
cv::putText(rgb, text, cv::Point(x, y + label_size.height),
cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 0, 0));
return 0;
}
static const int NDKCAMERAWINDOW_ID = 233;
int MyDisp::drawMat(JMat* mat,const char* text){
if(win==NULL)return 0;
//__android_log_print(ANDROID_LOG_WARN, "tooken", "drawmat");
int nv21_width = mat->width();
int nv21_height = mat->height();
int camera_orientation = 0;
int camera_facing = 0;
// resolve orientation from camera_orientation and accelerometer_sensor
{
if (!sensor_event_queue)
{
sensor_event_queue = ASensorManager_createEventQueue(sensor_manager, ALooper_prepare(ALOOPER_PREPARE_ALLOW_NON_CALLBACKS), NDKCAMERAWINDOW_ID, 0, 0);
ASensorEventQueue_enableSensor(sensor_event_queue, accelerometer_sensor);
}
int id = ALooper_pollAll(0, 0, 0, 0);
if (id == NDKCAMERAWINDOW_ID)
{
ASensorEvent e[8];
ssize_t num_event = 0;
while (ASensorEventQueue_hasEvents(sensor_event_queue) == 1)
{
num_event = ASensorEventQueue_getEvents(sensor_event_queue, e, 8);
if (num_event < 0)
break;
}
if (num_event > 0)
{
float acceleration_x = e[num_event - 1].acceleration.x;
float acceleration_y = e[num_event - 1].acceleration.y;
float acceleration_z = e[num_event - 1].acceleration.z;
// __android_log_print(ANDROID_LOG_WARN, "NdkCameraWindow", "x = %f, y = %f, z = %f", x, y, z);
if (acceleration_y > 7)
{
accelerometer_orientation = 0;
}
if (acceleration_x < -7)
{
accelerometer_orientation = 90;
}
if (acceleration_y < -7)
{
accelerometer_orientation = 180;
}
if (acceleration_x > 7)
{
accelerometer_orientation = 270;
}
}
}
}
int nv21_roi_x = 0;
int nv21_roi_y = 0;
int nv21_roi_w = 0;
int nv21_roi_h = 0;
int roi_x = 0;
int roi_y = 0;
int roi_w = 0;
int roi_h = 0;
int rotate_type = 0;
int render_w = 0;
int render_h = 0;
int render_rotate_type = 0;
{
int win_w = ANativeWindow_getWidth(win);
int win_h = ANativeWindow_getHeight(win);
if (accelerometer_orientation == 90 || accelerometer_orientation == 270)
{
std::swap(win_w, win_h);
}
const int final_orientation = (camera_orientation + accelerometer_orientation) % 360;
if (final_orientation == 0 || final_orientation == 180)
{
if (win_w * nv21_height > win_h * nv21_width)
{
roi_w = nv21_width;
roi_h = (nv21_width * win_h / win_w) / 2 * 2;
roi_x = 0;
roi_y = ((nv21_height - roi_h) / 2) / 2 * 2;
}
else
{
roi_h = nv21_height;
roi_w = (nv21_height * win_w / win_h) / 2 * 2;
roi_x = ((nv21_width - roi_w) / 2) / 2 * 2;
roi_y = 0;
}
nv21_roi_x = roi_x;
nv21_roi_y = roi_y;
nv21_roi_w = roi_w;
nv21_roi_h = roi_h;
}
if (final_orientation == 90 || final_orientation == 270)
{
if (win_w * nv21_width > win_h * nv21_height)
{
roi_w = nv21_height;
roi_h = (nv21_height * win_h / win_w) / 2 * 2;
roi_x = 0;
roi_y = ((nv21_width - roi_h) / 2) / 2 * 2;
}
else
{
roi_h = nv21_width;
roi_w = (nv21_width * win_w / win_h) / 2 * 2;
roi_x = ((nv21_height - roi_w) / 2) / 2 * 2;
roi_y = 0;
}
nv21_roi_x = roi_y;
nv21_roi_y = roi_x;
nv21_roi_w = roi_h;
nv21_roi_h = roi_w;
}
if (camera_facing == 0)
{
if (camera_orientation == 0 && accelerometer_orientation == 0)
{
rotate_type = 2;
}
if (camera_orientation == 0 && accelerometer_orientation == 90)
{
rotate_type = 7;
}
if (camera_orientation == 0 && accelerometer_orientation == 180)
{
rotate_type = 4;
}
if (camera_orientation == 0 && accelerometer_orientation == 270)
{
rotate_type = 5;
}
if (camera_orientation == 90 && accelerometer_orientation == 0)
{
rotate_type = 5;
}
if (camera_orientation == 90 && accelerometer_orientation == 90)
{
rotate_type = 2;
}
if (camera_orientation == 90 && accelerometer_orientation == 180)
{
rotate_type = 7;
}
if (camera_orientation == 90 && accelerometer_orientation == 270)
{
rotate_type = 4;
}
if (camera_orientation == 180 && accelerometer_orientation == 0)
{
rotate_type = 4;
}
if (camera_orientation == 180 && accelerometer_orientation == 90)
{
rotate_type = 5;
}
if (camera_orientation == 180 && accelerometer_orientation == 180)
{
rotate_type = 2;
}
if (camera_orientation == 180 && accelerometer_orientation == 270)
{
rotate_type = 7;
}
if (camera_orientation == 270 && accelerometer_orientation == 0)
{
rotate_type = 7;
}
if (camera_orientation == 270 && accelerometer_orientation == 90)
{
rotate_type = 4;
}
if (camera_orientation == 270 && accelerometer_orientation == 180)
{
rotate_type = 5;
}
if (camera_orientation == 270 && accelerometer_orientation == 270)
{
rotate_type = 2;
}
}
else
{
if (final_orientation == 0)
{
rotate_type = 1;
}
if (final_orientation == 90)
{
rotate_type = 6;
}
if (final_orientation == 180)
{
rotate_type = 3;
}
if (final_orientation == 270)
{
rotate_type = 8;
}
}
if (accelerometer_orientation == 0)
{
render_w = roi_w;
render_h = roi_h;
render_rotate_type = 1;
}
if (accelerometer_orientation == 90)
{
render_w = roi_h;
render_h = roi_w;
render_rotate_type = 8;
}
if (accelerometer_orientation == 180)
{
render_w = roi_w;
render_h = roi_h;
render_rotate_type = 3;
}
if (accelerometer_orientation == 270)
{
render_w = roi_h;
render_h = roi_w;
render_rotate_type = 6;
}
}
cv::Mat rgb;
cv::resize(mat->cvmat(),rgb,cv::Size(roi_w,roi_h),cv::INTER_LINEAR);
draw_fps(rgb);
if(text) draw_text(rgb,text);
cv::Mat rgb_render(render_h, render_w, CV_8UC3);
ncnn::kanna_rotate_c3(rgb.data, roi_w, roi_h, rgb_render.data, render_w, render_h, render_rotate_type);
ANativeWindow_setBuffersGeometry(win, render_w, render_h, AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM);
ANativeWindow_Buffer buf;
ANativeWindow_lock(win, &buf, NULL);
//__android_log_print(ANDROID_LOG_WARN, "tooken", "drawbuf");
// scale to target size
if (buf.format == AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM || buf.format == AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM)
{
for (int y = 0; y < render_h; y++)
{
const unsigned char* ptr = rgb_render.ptr<const unsigned char>(y);
unsigned char* outptr = (unsigned char*)buf.bits + buf.stride * 4 * y;
int x = 0;
#if __ARM_NEON
for (; x + 7 < render_w; x += 8)
{
uint8x8x3_t _rgb = vld3_u8(ptr);
uint8x8x4_t _rgba;
//_rgba.val[0] = _rgb.val[0];
//_rgba.val[1] = _rgb.val[1];
//_rgba.val[2] = _rgb.val[2];
_rgba.val[0] = _rgb.val[2];
_rgba.val[1] = _rgb.val[1];
_rgba.val[2] = _rgb.val[0];
_rgba.val[3] = vdup_n_u8(255);
vst4_u8(outptr, _rgba);
ptr += 24;
outptr += 32;
}
#endif // __ARM_NEON
for (; x < render_w; x++)
{
//outptr[0] = ptr[0];
//outptr[1] = ptr[1];
//outptr[2] = ptr[2];
outptr[0] = ptr[2];
outptr[1] = ptr[1];
outptr[2] = ptr[0];
outptr[3] = 255;
ptr += 3;
outptr += 4;
}
}
}
ANativeWindow_unlockAndPost(win);
return 0;
}
int MyDisp::set_window( ANativeWindow* newwin){
if (win) {
ANativeWindow_release(win);
}
win = newwin;
if(win) ANativeWindow_acquire(win);
return 0;
}

View File

@@ -0,0 +1,30 @@
#pragma once
#include <android/looper.h>
#include <android/native_window.h>
#include <android/sensor.h>
#include <media/NdkImageReader.h>
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <string>
#include <android/log.h>
#include "mat.h"
#include "jmat.h"
#include "benchmark.h"
class MyDisp {
private:
ANativeWindow *win = nullptr;
ASensorManager* sensor_manager;
mutable ASensorEventQueue* sensor_event_queue;
const ASensor* accelerometer_sensor;
int draw_text(cv::Mat& rgb,const char* text);
int draw_fps(cv::Mat& rgb);
public:
mutable int accelerometer_orientation;
int drawMat(JMat* mat,const char* text);
MyDisp();
virtual ~MyDisp();
int set_window( ANativeWindow* newwin);
int reset_fps();
};

View File

@@ -0,0 +1,428 @@
#include <android/asset_manager_jni.h>
#include <android/native_window_jni.h>
#include <android/native_window.h>
#include <android/log.h>
#include <jni.h>
#include <string>
#include <vector>
#include <unistd.h>
#include <platform.h>
#include <benchmark.h>
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include "scrfdncnn.h"
#if __ARM_NEON
#include <arm_neon.h>
#endif // __ARM_NEON
int MyDigit::checkfile(char* fn){
return 0;
}
int MyDigit::reset_inx(){
inx_wenet = 0;
return 0;
}
int MyDigit::picrst(const char* picfn,int* box,int index,const char* dumpfn){
if(!inited)return -999;
if(!mat_wenet)return -1;
if(index<0)return -2;
if(index>=cnt_wenet)return -3;
std::string picfile(picfn);
JMat onepic(picfile,1);
int* arr = onepic.tagarr();
arr[10] = box[0];
arr[11] = box[1];
arr[12] = box[2];
arr[13] = box[3];
//
float* pwenet = ai_wenet->nextbnf(mat_wenet,index);
if(!pwenet){
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "%d index error",index);
return -1;
}
JMat feat(256, 20, pwenet, 1);
//int* arr = onepic.tagarr();
double t0 = ncnn::get_current_time();
ai_munet->process(&onepic, arr, &feat);
double t1 = ncnn::get_current_time();
char text[1024];
float dist = t1-t0;
sprintf(text,"%d unet %f",index,dist);
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "%s dump %s",text,dumpfn);
drawMat(&onepic,text);
if(strlen(dumpfn)){
onepic.tojpg(dumpfn);
}
return 0;
}
int MyDigit::netrstpic(const char* picfn,int* box,int index,const char* dumpfn){
if(!inited)return -999;
if(!net_wavmat)return -1;
if(index<0)return -2;
if(index>net_wavmat->bnfblocks())return -3;
std::string picfile(picfn);
JMat onepic(picfile,1);
int* arr = onepic.tagarr();
arr[10] = box[0];
arr[11] = box[1];
arr[12] = box[2];
arr[13] = box[3];
//
JMat* mw = net_wavmat->bnfmat();
int* marr = mw->tagarr();
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "arr %d %d %d %d %d %d %d",marr[0],marr[1],marr[2],marr[3],marr[4],marr[5],marr[6]);
float* pwenet = ai_wenet->nextbnf(mw,index);
if(!pwenet){
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "%d index error",index);
return -1;
}
JMat feat(256, 20, pwenet, 1);
//int* arr = onepic.tagarr();
double t0 = ncnn::get_current_time();
ai_munet->process(&onepic, arr, &feat);
double t1 = ncnn::get_current_time();
char text[1024];
float dist = t1-t0;
sprintf(text,"%d unet %f",index,dist);
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "%s dump %s",text,dumpfn);
drawMat(&onepic,text);
if(strlen(dumpfn)){
onepic.tojpg(dumpfn);
}
return 0;
}
int MyDigit::netrst(int index,const char* dumpfn){
if(!inited)return -999;
if(!net_wavmat)return -1;
if(index<0)return -2;
if(index>net_wavmat->bnfblocks())return -3;
JMat onepic = mat_pic->clone();
JMat* mw = net_wavmat->bnfmat();
float* pwenet = ai_wenet->nextbnf(mw,index);
if(!pwenet){
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "%d index error",index);
return -1;
}
//float* pd = pwenet+256*20*index;
JMat feat(256, 20, pwenet, 1);
int* arr = mat_pic->tagarr();
double t0 = ncnn::get_current_time();
ai_munet->process(&onepic, arr, &feat);
double t1 = ncnn::get_current_time();
char text[1024];
float dist = t1-t0;
sprintf(text,"%d unet %f",index,dist);
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "%s dump %s",text,dumpfn);
drawMat(&onepic,text);
if(strlen(dumpfn)){
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "%s dump %s",text,dumpfn);
onepic.tojpg(dumpfn);
}
return 0;
}
int MyDigit::newrst(int index,const char* dumpfn){
if(!inited)return -999;
if(!mat_wenet)return -1;
if(index<0)return -2;
if(index>=cnt_wenet)return -3;
if(!mat_pic)return -4;
JMat onepic = mat_pic->clone();
float* pwenet = ai_wenet->nextbnf(mat_wenet,index);
if(!pwenet){
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "%d index error",index);
return -1;
}
//float* pd = pwenet+256*20*index;
JMat feat(256, 20, pwenet, 1);
int* arr = mat_pic->tagarr();
double t0 = ncnn::get_current_time();
ai_munet->process(&onepic, arr, &feat);
double t1 = ncnn::get_current_time();
char text[1024];
float dist = t1-t0;
sprintf(text,"%d unet %f",index,dist);
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "%s dump %s",text,dumpfn);
drawMat(&onepic,text);
if(strlen(dumpfn)){
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "%s dump %s",text,dumpfn);
onepic.tojpg(dumpfn);
}
return 0;
}
int MyDigit::drawpic(const char* picfn){
if(!inited)return -999;
std::string picfile(picfn);
JMat mat(picfile,1);
//
drawMat(&mat,NULL);
return 0;
}
int MyDigit::newpic(const char* picfn,const char* dumpbox){
if(!inited)return -999;
if(mat_pic){
delete mat_pic;
mat_pic = nullptr;
}
std::string picfile(picfn);
mat_pic = new JMat(picfile,1);
int* arr = mat_pic->tagarr();
double t0 = ncnn::get_current_time();
ai_scrfd->detect(mat_pic,arr);
double t1 = ncnn::get_current_time();
if((arr[0]<=0)||(arr[1]<=0))return -1;
int* apts = arr+64;
ai_pfpld->detect(mat_pic,arr,apts);
double t2 = ncnn::get_current_time();
char text[1024];
float dist1 = t1-t0;
float dist2 = t2-t1;
sprintf(text,"%s scrfd %f pfpld %f",picfn,dist1,dist2);
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "%s",text);
drawMat(mat_pic,text);
if(apts[0]<=0)return -2;
if(strlen(dumpbox)){
FILE* file = fopen(dumpbox, "w");
if(file){
fprintf(file,"face w %d h %d\n",arr[0],arr[1]);
int* box = arr+2;
fprintf(file,"orig box x1y1 %d %d x2y2 %d %d\n",box[0],box[1],box[2],box[3]);
box = arr+6;
fprintf(file,"scrfd box x1y1 %d %d x2y2 %d %d\n",box[0],box[1],box[2],box[3]);
box = arr+10;
fprintf(file,"pfpld box x1y1 %d %d x2y2 %d %d\n",box[0],box[1],box[2],box[3]);
fclose(file);
}
}
return 0;
}
int MyDigit::netwav(const char* url,float duration){
if(!inited)return -999;
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "netwav %s",url);
if(net_curl){
net_curl->cancel();
delete net_curl;
net_curl = nullptr;
}
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "netwav gogogo");
if(net_wavmat){
delete net_wavmat;
net_wavmat = nullptr;
}
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "netwav %f",duration);
net_wavmat = new KWav(duration);
net_curl = new NetCurl((char*)url,duration,net_wavmat);
for(int k=0;k<20;k++){
if(net_curl->checked())break;
usleep(100000);
}
int rst = net_wavmat->bnfblocks();//net_curl->checked();
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "netwav %d",rst);
return rst;
}
int MyDigit::newwav(const char* wavfn,const char* dumpfn){
if(!inited)return -999;
if(mat_wenet){
delete mat_wenet;
mat_wenet = nullptr;
}
double t0 = ncnn::get_current_time();
int rst = ai_wenet->nextwav(wavfn,&mat_wenet);
cnt_wenet = rst;
double t1 = ncnn::get_current_time();
JMat mat(640,480,CV_8UC3);
char text[1024];
float dist = t1-t0;
sprintf(text,"%s wenet %f",wavfn,dist);
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "%s",text);
//drawMat(&mat,text);
if(strlen(dumpfn)){
mat_wenet->tobin(dumpfn);
}
if(!mat_wenet)return -1;
inx_wenet = 0;
return rst;
}
int MyDigit::initModel(const char* modeldir){
if(inited)return 0;
guiji_curl_init(modeldir);
double t0 = ncnn::get_current_time();
ai_wenet = new Wenet(modeldir,"wenet");
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "init wenet");
double t1 = ncnn::get_current_time();
ai_scrfd = new Scrfd(modeldir,"scrfd_500m_kps-opt2",1080,1920);
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "init scrfd");
double t2 = ncnn::get_current_time();
ai_pfpld = new Pfpld(modeldir,"pfpld",1080,1920);
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "init pfpld");
double t3 = ncnn::get_current_time();
ai_munet = new Mobunet(modeldir,"mobileunet_v5_wenet_sim");
double t4 = ncnn::get_current_time();
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "init munet");
JMat mat(640,480,CV_8UC3);
char text[1024];
float da = t1-t0;
float db = t2-t1;
float dc = t3-t2;
float dd = t4-t3;
sprintf(text,"%s model wenet %f scrfd %f pfpld %f unet %f",modeldir,da,db,dc,dd);
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "%s",text);
drawMat(&mat,text);
//
init_wenetloop(ai_wenet);
inited = true;
return 0;
}
MyDigit::MyDigit(){
}
MyDigit::~MyDigit(){
if(inited){
final_wenetloop();
if(mat_wenet){
delete mat_wenet;
mat_wenet = nullptr;
}
if(net_curl){
delete net_curl;
net_curl = nullptr;
}
if(net_wavmat){
delete net_wavmat;
net_wavmat = nullptr;
}
delete ai_wenet;
delete ai_scrfd;
delete ai_pfpld;
delete ai_munet;
}
}
static MyDigit* g_digit = 0;
extern "C" {
JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *reserved) {
__android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnLoad");
g_digit = new MyDigit;
return JNI_VERSION_1_4;
}
JNIEXPORT void JNI_OnUnload(JavaVM *vm, void *reserved) {
__android_log_print(ANDROID_LOG_DEBUG, "ncnn", "JNI_OnUnload");
delete g_digit;
g_digit = 0;
}
static std::string getStringUTF(JNIEnv *env, jstring obj) {
char *c_str = (char *) env->GetStringUTFChars(obj, nullptr);
std::string tmpString = std::string(c_str);
env->ReleaseStringUTFChars(obj, c_str);
return tmpString;
}
JNIEXPORT jint JNICALL
Java_com_btows_ncnntest_SCRFDNcnn_initModel(JNIEnv *env, jobject thiz, jstring path){
std::string str = getStringUTF(env,path);
return g_digit->initModel(str.c_str());
}
JNIEXPORT jint JNICALL
Java_com_btows_ncnntest_SCRFDNcnn_onewav(JNIEnv *env, jobject thiz, jstring wavfn,jstring dumpfn){
std::string s_wav = getStringUTF(env,wavfn);
std::string s_dump = getStringUTF(env,dumpfn);
return g_digit->newwav(s_wav.c_str(),s_dump.c_str());
}
JNIEXPORT jint JNICALL
Java_com_btows_ncnntest_SCRFDNcnn_netwav(JNIEnv *env, jobject thiz, jstring wavurl,jfloat duration){
std::string s_wav = getStringUTF(env,wavurl);
return g_digit->netwav(s_wav.c_str(),duration);
}
JNIEXPORT jint JNICALL
Java_com_btows_ncnntest_SCRFDNcnn_onepic(JNIEnv *env, jobject thiz, jstring picfn,jstring dumpfn){
std::string s_pic = getStringUTF(env,picfn);
std::string s_dump = getStringUTF(env,dumpfn);
return g_digit->newpic(s_pic.c_str(),s_dump.c_str());
}
JNIEXPORT jint JNICALL
Java_com_btows_ncnntest_SCRFDNcnn_drawpic(JNIEnv *env, jobject thiz, jstring picfn){
std::string s_pic = getStringUTF(env,picfn);
return g_digit->drawpic(s_pic.c_str());
}
// public native boolean openCamera(int facing);
JNIEXPORT jint JNICALL Java_com_btows_ncnntest_SCRFDNcnn_onerst(JNIEnv* env, jobject thiz, jint index,jstring dumpfn) {
std::string s_dump = getStringUTF(env,dumpfn);
return g_digit->newrst(index,s_dump.c_str());
}
JNIEXPORT jint JNICALL Java_com_btows_ncnntest_SCRFDNcnn_netrst(JNIEnv* env, jobject thiz, jint index,jstring dumpfn) {
std::string s_dump = getStringUTF(env,dumpfn);
return g_digit->netrst(index,s_dump.c_str());
}
JNIEXPORT jint JNICALL Java_com_btows_ncnntest_SCRFDNcnn_picrst(JNIEnv* env, jobject thiz,jstring picfile, jintArray arrbox, jint index,jstring dumpfn){
std::string s_pic = getStringUTF(env,picfile);
std::string s_dump = getStringUTF(env,dumpfn);
jint *boxData = (jint*) env->GetIntArrayElements( arrbox, NULL);
int rst = g_digit->picrst(s_pic.c_str(),(int*)boxData,index,s_dump.c_str());
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "picrst %d",rst);
env->ReleaseIntArrayElements( arrbox, boxData, 0);
return rst;
}
JNIEXPORT jint JNICALL Java_com_btows_ncnntest_SCRFDNcnn_netrstpic(JNIEnv* env, jobject thiz,jstring picfile, jintArray arrbox, jint index,jstring dumpfn){
std::string s_pic = getStringUTF(env,picfile);
std::string s_dump = getStringUTF(env,dumpfn);
jint *boxData = (jint*) env->GetIntArrayElements( arrbox, NULL);
int rst = g_digit->netrstpic(s_pic.c_str(),(int*)boxData,index,s_dump.c_str());
__android_log_print(ANDROID_LOG_DEBUG, "tooken", "picrst %d",rst);
env->ReleaseIntArrayElements( arrbox, boxData, 0);
return rst;
}
JNIEXPORT jint JNICALL Java_com_btows_ncnntest_SCRFDNcnn_reset(JNIEnv* env, jobject thiz){
//g_digit->reset_fps();
//g_digit->reset_inx();
g_digit->set_window(NULL);
return 0;
}
// public native boolean setOutputWindow(Surface surface);
JNIEXPORT jint JNICALL Java_com_btows_ncnntest_SCRFDNcnn_setOutputWindow(JNIEnv* env, jobject thiz, jobject surface)
{
ANativeWindow* win = ANativeWindow_fromSurface(env, surface);
__android_log_print(ANDROID_LOG_DEBUG, "ncnn", "setOutputWindow %p", win);
g_digit->set_window(win);
return 0;
}
}

View File

@@ -0,0 +1,50 @@
#pragma once
#include "jmat.h"
#include "kmat.h"
#include "scrfd.h"
#include "pfpld.h"
#include "munet.h"
#include "wenet.h"
#include "utils/face_utils.h"
#include "netcurl.h"
#include "looper.h"
#include "ndkwin.h"
#include "netwav.h"
#include "netcurl.h"
class MyDigit:public MyDisp {
private:
Scrfd* ai_scrfd = nullptr;
Pfpld* ai_pfpld = nullptr;
Wenet* ai_wenet = nullptr;
Mobunet* ai_munet = nullptr;
int inited = false;
JMat* mat_wenet = nullptr;
int cnt_wenet = 0;
int inx_wenet = 0;
JMat* mat_pic = nullptr;
JMat* mat_bg = nullptr;
int checkfile(char* fn);
NetCurl* net_curl = nullptr;
KWav* net_wavmat = nullptr;
public:
MyDigit();
virtual ~MyDigit();
int initModel(const char* modeldir);
int netwav(const char* url,float duration);
int netrst(int index,const char* dumpfn);
int netrstpic(const char* picfn,int* box,int index,const char* dumpfn);
int newwav(const char* wavfn,const char* dumpfn);
int newpic(const char* picfn,const char* dumpbox);
int drawpic(const char* picfn);
int newrst(int index,const char* dumpfn);
int picrst(const char* picfn,int* box,int index,const char* dumpfn);
int reset_inx();
};

View File

@@ -0,0 +1,21 @@
#include <Log.h>
#include "AudioTrack.h"
#define TAG "AudioTrack"
AudioTrack::AudioTrack(CommObj* obj) {
audioTrackObj = obj;//
}
AudioTrack::~AudioTrack() = default;
void AudioTrack::start(int sampleRate, int sampleFormat, int channels, int bytesPerSample) {
LOGI(TAG, "start %d %d %d %d", sampleRate, sampleFormat, channels, bytesPerSample);
}
int AudioTrack::write(uint8_t *buffer, int size) {
return -1;
}
void AudioTrack::stop() {
}

View File

@@ -0,0 +1,24 @@
#ifndef GPLAYER_AUDIOTRACK_H
#define GPLAYER_AUDIOTRACK_H
#include "CommObj.h"
#include <stdint.h>
class AudioTrack {
public:
AudioTrack(CommObj* obj);
virtual ~AudioTrack();
void start(int sampleRate, int sampleFormat, int channels, int bytesPerSample);
int write(uint8_t *buffer, int size);
void stop();
private:
CommObj* audioTrackObj;
};
#endif

View File

@@ -0,0 +1,46 @@
#include <LoopThread.h>
#include "BaseRenderHelper.h"
BaseRenderHelper::BaseRenderHelper(FrameSource *source, MessageSource *messageSource, int mediaCodecFlag, bool hasAudio, bool hasVideo) {
this->hasAudio = hasAudio;
this->hasVideo = hasVideo;
this->messageSource = messageSource;
}
BaseRenderHelper::~BaseRenderHelper() {
}
void BaseRenderHelper::setNativeWindow(NativeWindowType window) {
nativeWindow = window;
}
void BaseRenderHelper::setAudioTrack(AudioTrack *track) {
}
void BaseRenderHelper::setVideoParams(int width, int height) {
videoWidth = width;
videoHeight = height;
}
void BaseRenderHelper::setAudioParams(int sampleRate, int channels, int format, int bytesPerSample) {
}
void BaseRenderHelper::initAudioRenderer() {
}
void BaseRenderHelper::initVideoRenderer() {
}
int BaseRenderHelper::renderAudio(int arg1, long arg2) {
return 0;
}
int BaseRenderHelper::renderVideo(int arg1, long arg2) {
return 0;
}
void BaseRenderHelper::releaseAudioRenderer() {
}
void BaseRenderHelper::releaseVideoRenderer() {
}

View File

@@ -0,0 +1,54 @@
#ifndef GPLAYER_BASERENDERHELPER_H
#define GPLAYER_BASERENDERHELPER_H
#include "MessageSource.h"
#include "AudioTrack.h"
#include "FrameSource.h"
#define USE_RENDER
#ifdef USE_RENDER
#include <EGL/egl.h>
#else
typedef void* NativeWindowType;
#endif
class BaseRenderHelper {
public:
BaseRenderHelper(FrameSource *source, MessageSource *messageSource, int mediaCodecFlag, bool hasAudio, bool hasVideo);
virtual ~BaseRenderHelper();
virtual void setNativeWindow(NativeWindowType window);
virtual void setAudioTrack(AudioTrack *track);
virtual void setVideoParams(int width, int height);
virtual void setAudioParams(int sampleRate, int channels, int format, int bytesPerSample);
virtual void initAudioRenderer();
virtual void initVideoRenderer();
virtual int renderAudio(int arg1, long arg2);
virtual int renderVideo(int arg1, long arg2);
virtual void releaseAudioRenderer();
virtual void releaseVideoRenderer();
protected:
MessageSource *messageSource = nullptr;
NativeWindowType nativeWindow = 0;
int videoWidth = 0;
int videoHeight = 0;
bool hasAudio = false;
bool hasVideo = false;
};
#endif //GPLAYER_RENDERHELPER_H

View File

@@ -0,0 +1,19 @@
#ifndef GPLAYER_COMMOBJ_H_
#define GPLAYER_COMMOBJ_H_
#ifdef ANDROID
#include <jni.h>
typedef jobject CommObj;
#else
class CommObj{};
#endif
class MessageCb{
public:
MessageCb(){};
virtual ~MessageCb(){};
virtual void onMessageCallback(int msgId, int arg1, long arg2, const char *msg1, const char *msg2){};
virtual void onMessageCallback(int msgId, int arg1, long arg2, const char *msg1, const char *msg2, CommObj obj){};
};
#endif

View File

@@ -0,0 +1,165 @@
#ifndef GPLAYER_CONCURRENTQUEUE_H
#define GPLAYER_CONCURRENTQUEUE_H
#include <deque>
#include <mutex>
#include <Log.h>
#include <condition_variable>
template<typename T>
class ConcurrentQueue {
public:
ConcurrentQueue(int maxSize, std::string,int simp=0);
~ConcurrentQueue();
unsigned long push(T data);
unsigned long front(T *data);
void pop();
void flush();
unsigned long size();
void reset();
virtual void deleteItem(T data);
void log(const char *msg);
private:
unsigned long maxSize;
int m_simp;
std::string tag;
std::deque<T> queue;
volatile bool isWaitingIn;
std::mutex inConLock;
std::condition_variable inConVar;
volatile bool isWaitingOut;
std::mutex outConLock;
std::condition_variable outConVar;
std::mutex queueLock;
};
template<typename T>
ConcurrentQueue<T>::ConcurrentQueue(int maxSize, std::string tag,int simp) {
this->maxSize = maxSize > 1 ? maxSize : 1;
this->tag = tag;
this->m_simp = simp;
}
template<typename T>
ConcurrentQueue<T>::~ConcurrentQueue() {
}
template<typename T>
unsigned long ConcurrentQueue<T>::push(T data) {
if(m_simp){
if (size() >= maxSize) {
deleteItem(data);
return 0;
}
queueLock.lock();
queue.push_back(data);
queueLock.unlock();
return size();
}
queue.push_back(data);
if (isWaitingOut) {
log("output notify all");
outConVar.notify_all();
}
if (size() >= maxSize) {
isWaitingIn = true;
log("input waiting start");
std::unique_lock<std::mutex> lck(inConLock);
inConVar.wait_for(lck, std::chrono::milliseconds(200));
log("input waiting end");
isWaitingIn = false;
}
return size();
}
template<typename T>
unsigned long ConcurrentQueue<T>::front(T *data) {
if(m_simp){
if (size() <= 0) return 0;
*data = queue.front();
if (!(*data)) {
pop();
return 0;
}
return size();
}
if (size() <= 0) {
isWaitingOut = true;
log("output waiting start");
std::unique_lock<std::mutex> lck(outConLock);
outConVar.wait_for(lck, std::chrono::milliseconds(200));
log("output waiting end");
isWaitingOut = false;
return 0;
}
*data = queue.front();
if (!(*data)) {
pop();
return 0;
}
return size();
}
template<typename T>
void ConcurrentQueue<T>::pop() {
queueLock.lock();
if (size() > 0) {
T data = queue.front();
if(!m_simp)deleteItem(data);
queue.pop_front();
}
queueLock.unlock();
if(m_simp)return;
if (isWaitingIn) {
log("input notify all");
inConVar.notify_all();
}
}
template<typename T>
void ConcurrentQueue<T>::flush() {
queueLock.lock();
while (size() > 0) {
T data = queue.front();
deleteItem(data);
queue.pop_front();
}
queueLock.unlock();
}
template<typename T>
unsigned long ConcurrentQueue<T>::size() {
return queue.size();
}
template<typename T>
void ConcurrentQueue<T>::reset() {
inConVar.notify_all();
outConVar.notify_all();
}
template<typename T>
void ConcurrentQueue<T>::deleteItem(T data) {
delete data;
}
template<typename T>
void ConcurrentQueue<T>::log(const char *msg) {
LOGD(tag.c_str(), msg);
}
#endif //GPLAYER_CONCURRENTQUEUE_H

View File

@@ -0,0 +1,104 @@
#include <Log.h>
#include <thread>
#include "FrameSource.h"
#define TAG "FrameSource"
FrameSource::FrameSource(int audioMaxSize, int videoMaxSize) {
LOGI(TAG, "CoreFlow : create FrameSource");
audioPacketQueue = new ConcurrentQueue<MediaData*>(audioMaxSize, "AudioFrameQueue");
videoPacketQueue = new ConcurrentQueue<MediaData*>(videoMaxSize, "VideoFrameQueue");
videoRecyleQueue = new ConcurrentQueue<JMat*>(videoMaxSize, "VideoRecyleQueue",1);
}
FrameSource::~FrameSource() {
LOGI(TAG, "CoreFlow : FrameSource destroyed %d %d",
audioPacketQueue->size(), videoPacketQueue->size());
delete audioPacketQueue;
audioPacketQueue = nullptr;
delete videoPacketQueue;
videoPacketQueue = nullptr;
delete videoRecyleQueue;
videoRecyleQueue = nullptr;
}
unsigned long FrameSource::pushAudFrame(MediaData *frame) {
LOGD(TAG, "pushAudFrame %lld", frame->pts);
auto desBuffer = new MediaData();
//MediaHelper::copy(frame, desBuffer);
desBuffer->copy(frame);
return audioPacketQueue->push(desBuffer);
}
unsigned long FrameSource::pushVidFrame(MediaData *frame) {
LOGD(TAG, "pushVidFrame %lld", frame->pts);
auto desBuffer = new MediaData();
//MediaHelper::copy(frame, desBuffer);
desBuffer->copy(frame);
return videoPacketQueue->push(desBuffer);
}
unsigned long FrameSource::pushVidRecyle(JMat *frame) {
if(!frame)return 0;
return videoRecyleQueue->push(frame);
}
unsigned long FrameSource::readAudFrame(MediaData **frame) {
unsigned long size = audioPacketQueue->front(frame);
if (size > 0) {
LOGD(TAG, "readAudFrame %lld", (*frame)->pts);
}
return size;
}
unsigned long FrameSource::readVidFrame(MediaData **frame) {
unsigned long size = videoPacketQueue->front(frame);
if (size > 0) {
LOGD(TAG, "readVidFrame %lld", (*frame)->pts);
}
return size;
}
unsigned long FrameSource::popVidRecyle(JMat **frame) {
unsigned long size = videoRecyleQueue->front(frame);
if (size > 0) {
videoRecyleQueue->pop();
}
return size;
}
void FrameSource::popAudFrame(MediaData *frame) {
LOGD(TAG, "popAudFrame %lld", frame->pts);
audioPacketQueue->pop();
}
void FrameSource::popVidFrame(MediaData *frame) {
LOGD(TAG, "popVidFrame %lld", frame->pts);
videoPacketQueue->pop();
}
void FrameSource::flush() {
audioPacketQueue->flush();
videoPacketQueue->flush();
videoRecyleQueue->flush();
LOGI(TAG, "flushBuffer");
}
void FrameSource::reset() {
audioPacketQueue->reset();
videoPacketQueue->reset();
videoRecyleQueue->reset();
}
unsigned long FrameSource::audioSize() {
return audioPacketQueue->size();
}
unsigned long FrameSource::videoSize() {
return videoPacketQueue->size();
}
unsigned long FrameSource::recyleSize() {
return videoRecyleQueue->size();
}

View File

@@ -0,0 +1,48 @@
#ifndef GPLAYER_FRAMESOURCE_H
#define GPLAYER_FRAMESOURCE_H
#include "MediaData.h"
#include "ConcurrentQueue.h"
class FrameSource {
public:
FrameSource(int audioMaxSize, int videoMaxSize);
~FrameSource();
public:
unsigned long pushAudFrame(MediaData *frame);
unsigned long pushVidFrame(MediaData *frame);
unsigned long pushVidRecyle(JMat *frame);
unsigned long popVidRecyle(JMat **frame);
unsigned long readAudFrame(MediaData **frame);
unsigned long readVidFrame(MediaData **frame);
void popAudFrame(MediaData *frame);
void popVidFrame(MediaData *frame);
void flush();
void reset();
unsigned long audioSize();
unsigned long videoSize();
unsigned long recyleSize();
private:
ConcurrentQueue<MediaData *> *audioPacketQueue;
ConcurrentQueue<MediaData *> *videoPacketQueue;
ConcurrentQueue<JMat *> *videoRecyleQueue;
};
#endif //GPLAYER_FRAMESOURCE_H

View File

@@ -0,0 +1,80 @@
#if defined(_WIN32)
#define _CRT_SECURE_NO_WARNINGS
#endif
#include "Log.h"
#include <stdio.h>
#include <time.h>
#include <stdarg.h>
#ifdef __ANDROID__
#include <android/log.h>
android_LogPriority s_android_logprio[LOG_TRACE + 1] = {
ANDROID_LOG_UNKNOWN,
ANDROID_LOG_FATAL,
ANDROID_LOG_ERROR,
ANDROID_LOG_WARN,
ANDROID_LOG_INFO,
ANDROID_LOG_DEBUG,
ANDROID_LOG_VERBOSE
};
#endif
#if defined(_WIN32)
#include <windows.h>
#endif
void __log_print(int lv, const char *tag, const char *funame, int line, const char *fmt, ...) {
char log_info[2040];
char *buf = log_info;
int ret, len = sizeof(log_info);
//Android 不需要时间
#ifndef __ANDROID__
/*
if (lv <= LogLevel::LOG_INFO) { // 日志级别不小于INFO则打印时带时间标记
*buf++ = '[';
_get_curtime_str(buf);
//buf = buf + strlen(buf);
buf += 23; // 时间格式为XXXX - XX - XX XX : XX : XX.XXX 共占23个字节
*buf++ = ']';
*buf++ = ' ';
len -= buf - log_info;
}
*/
if (lv <= LogLevel::LOG_WARN) { // 日志级别不小于WARN则打印时带代码行信息
ret = sprintf(buf, "%s line:%-4d ", funame, line);
buf += ret;
len -= ret;
}
#endif
va_list arglist;
va_start(arglist, fmt);
int itemLen = buf - log_info;
#if defined( WIN32 )
ret = _vsnprintf(buf, len - 1, fmt, arglist);
#else
ret = vsnprintf(buf, len - 1, fmt, arglist);
#endif
if (ret < 0) {
buf[len - 1] = 0;
buf[len - 2] = '\n';
itemLen += len - 1;
} else
itemLen += ret;
va_end(arglist);
#if defined(__ANDROID__)
__android_log_print(s_android_logprio[lv], tag, log_info, "");
#else
//本地输出
//printf("Tag=%s %s\n", tag, log_info);
#endif
}

View File

@@ -0,0 +1,43 @@
#ifndef __GPLAYER_LOG_H__
#define __GPLAYER_LOG_H__
#ifdef __cplusplus
extern "C" {
#endif
//调试日志开关1为开其它为关
#define LOG_OPEN 0
enum LogLevel
{
LOG_OFF = 0, //!< 不打印日志
LOG_FATAL = 1, //!< 严重
LOG_ERROR = 2, //!< 错误
LOG_WARN = 3, //!< 警告
LOG_INFO = 4, //!< 信息
LOG_DEBUG = 5, //!< 调试
LOG_TRACE = 6, //!< 跟踪
};
void __log_print(int lv, const char* tag, const char* funame, int line, const char *fmt, ...);
#define LOGI(TAG, ...) __log_print(LogLevel::LOG_INFO, TAG, __FUNCTION__, __LINE__, __VA_ARGS__)
#define LOGW(TAG, ...) __log_print(LogLevel::LOG_WARN, TAG, __FUNCTION__, __LINE__, __VA_ARGS__)
#define LOGE(TAG, ...) __log_print(LogLevel::LOG_ERROR, TAG, __FUNCTION__, __LINE__, __VA_ARGS__)
#define LOGF(TAG, ...) __log_print(LogLevel::LOG_FATAL, TAG, __FUNCTION__, __LINE__, __VA_ARGS__)
#if defined(__ANDROID__)
#if(LOG_OPEN == 1)
#define LOGD(TAG,...) __log_print(LogLevel::LOG_DEBUG, TAG, __FUNCTION__, __LINE__, __VA_ARGS__)
#else
#define LOGD(TAG, ...) NULL
#endif
#else
#define LOGD(TAG, ...) __log_print(LogLevel::LOG_DEBUG, TAG, __FUNCTION__, __LINE__, __VA_ARGS__)
#endif
#ifdef __cplusplus
};
#endif
#endif // !__GPLAYER_LOG_H__

View File

@@ -0,0 +1,84 @@
/*
* Created by Gibbs on 2021/1/1.
* Copyright (c) 2021 Gibbs. All rights reserved.
*/
#include "LoopThread.h"
#include <utility>
LoopThread::LoopThread() {
setFunction(std::bind(&LoopThread::handleRunning, this));
}
LoopThread::~LoopThread()= default;
void LoopThread::handleRunning() {
if (startFunc) {
startFunc();
}
if (notifyFunc) {
notifyFunc(NOTIFY_START);
}
isStarted = true;
while (mRunning) {
if (mPausing) {
std::unique_lock<std::mutex> lck(threadLock);
conVar.wait(lck);
continue;
}
if (!updateFunc) {
break;
}
bool hasParams = arg1 >= 0 || arg2 >= 0;
int updateResult = updateFunc(arg1, arg2);
if (hasParams) {
arg1 = -1;
arg2 = -1;
}
if (updateResult == ERROR_EXIST) {
mRunning = false;
continue;
} else if (updateResult == ERROR_PAUSE) {
mPausing = true;
continue;
}
}
if (endFunc) {
endFunc();
}
if (notifyFunc) {
notifyFunc(NOTIFY_END);
}
isStarted = false;
}
void LoopThread::setStartFunc(std::function<void(void)> func) {
startFunc = std::move(func);
}
void LoopThread::setUpdateFunc(std::function<int(int, long)> func) {
updateFunc = std::move(func);
}
void LoopThread::setEndFunc(std::function<void(void)> func) {
endFunc = std::move(func);
}
void LoopThread::setNotifyFunc(std::function<void(int)> func) {
notifyFunc = std::move(func);
}
void LoopThread::resume() {
XThread::resume();
conVar.notify_all();
}
bool LoopThread::stop() {
if (isPausing()) {
resume();
}
return XThread::stop();
}

View File

@@ -0,0 +1,60 @@
#ifndef GPLAYER_LOOPTHREAD_H
#define GPLAYER_LOOPTHREAD_H
#include "XThread.h"
#include <functional>
#include <mutex>
#include <condition_variable>
#define ERROR_EXIST -100
#define ERROR_PAUSE -101
#define NOTIFY_START 0
#define NOTIFY_END 1
using namespace std;
class LoopThread : public XThread {
public:
LoopThread();
virtual ~LoopThread();
void setStartFunc(std::function<void(void)> func);
void setUpdateFunc(std::function<int(int, long)> func);
void setEndFunc(std::function<void(void)> func);
void setNotifyFunc(std::function<void(int)> func);
void resume() override ;
bool stop() override ;
bool hasStarted() {
return isStarted;
}
void setArgs(int a, long b) {
this->arg1 = a;
this->arg2 = b;
}
protected:
void handleRunning();
private:
std::function<void(void)> startFunc;
std::function<int(int, long)> updateFunc;
std::function<void(void)> endFunc;
std::function<void(int)> notifyFunc;
bool isStarted = false;
std::mutex threadLock;
std::condition_variable conVar;
int arg1 = -1;
long arg2 = -1;
};
#endif //GPLAYER_LOOPTHREAD_H

View File

@@ -0,0 +1,43 @@
/*
* Created by Gibbs on 2021/1/1.
* Copyright (c) 2021 Gibbs. All rights reserved.
*/
#include "LoopThreadHelper.h"
LoopThread *
LoopThreadHelper::createLoopThread(const std::function<int(int, long)>& updateFunc) {
return LoopThreadHelper::createLoopThread(nullptr, updateFunc, nullptr);
}
LoopThread *LoopThreadHelper::createLoopThread(const std::function<int(int, long)>& updateFunc,
const std::function<void(int)>& notifyFunc) {
return LoopThreadHelper::createLoopThread(nullptr, updateFunc, nullptr, notifyFunc);
}
LoopThread *LoopThreadHelper::createLoopThread(const std::function<void(void)>& startFunc,
const std::function<int(int, long)>& updateFunc,
const std::function<void()>& endFunc) {
return LoopThreadHelper::createLoopThread(startFunc, updateFunc, endFunc, nullptr);
}
LoopThread *LoopThreadHelper::createLoopThread(const std::function<void(void)>& startFunc,
const std::function<int(int, long)>& updateFunc,
const std::function<void()>& endFunc,
const std::function<void(int)>& notifyFunc) {
auto *thread = new LoopThread();
if (startFunc != nullptr) {
thread->setStartFunc(startFunc);
}
if (updateFunc != nullptr) {
thread->setUpdateFunc(updateFunc);
}
if (endFunc != nullptr) {
thread->setEndFunc(endFunc);
}
if (notifyFunc != nullptr) {
thread->setNotifyFunc(notifyFunc);
}
thread->start();
return thread;
}

View File

@@ -0,0 +1,26 @@
#ifndef GPLAYER_LOOPTHREADHELPER_H
#define GPLAYER_LOOPTHREADHELPER_H
#include "LoopThread.h"
#include <functional>
class LoopThreadHelper {
public:
static LoopThread *createLoopThread(const std::function<int(int, long)>& updateFunc);
static LoopThread *createLoopThread(const std::function<int(int, long)>& updateFunc,
const std::function<void(int)>& notifyFunc);
static LoopThread *createLoopThread(const std::function<void(void)>& startFunc,
const std::function<int(int, long)>& updateFunc,
const std::function<void(void)>& endFunc);
static LoopThread *createLoopThread(const std::function<void(void)>& startFunc,
const std::function<int(int, long)>& updateFunc,
const std::function<void(void)>& endFunc,
const std::function<void(int)>& notifyFunc);
};
#endif //GPLAYER_LOOPTHREADHELPER_H

View File

@@ -0,0 +1,87 @@
/*
* Created by Gibbs on 2021/1/1.
* Copyright (c) 2021 Gibbs. All rights reserved.
*/
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <Log.h>
#include "MediaData.h"
MediaData::MediaData() = default;
MediaData::MediaData(JMat* mat,bool ref ){
m_data = mat;
data = mat->udata();
size = mat->size();
data1 = nullptr;
size1 = 0;
data2 = nullptr;
size2 = 0;
width = mat->width();;
height = mat->height();
m_ref = ref;
}
MediaData::MediaData(JMat* fmat,JMat* mmat,JMat* bmat,bool ref ){
m_data = fmat;
m_msk = mmat;
m_bg = bmat;
data = fmat->udata();
size = fmat->size();
data1 = mmat->udata();
size1 = mmat->size();
data2 = bmat->udata();
size2 = bmat->size();
width = fmat->width();;
height = fmat->height();
m_ref = ref;
}
int MediaData::copy(MediaData* mediadata){
//copy
MediaData* mmm = mediadata;
data = mmm->data;
size = mmm->size;
data1 = mmm->data1;
size1 = mmm->size1;
data2 = mmm->data2;
size2 = mmm->size2;
width = mmm->width;
height = mmm->height;
pts = mediadata->pts;
dts = mediadata->dts;
m_data = mmm->m_data;
m_msk = mmm->m_msk;
m_bg = mmm->m_bg;
m_ref = true;
mmm->reset();
return 0;
}
MediaData::~MediaData() {
if(m_data){
delete m_data;
m_data = nullptr;
}
if(m_msk){
delete m_msk;
m_msk = nullptr;
}
if(m_bg){
delete m_bg;
m_bg = nullptr;
}
}
void MediaData::reset(int flag){
m_data = nullptr;
m_msk = nullptr;
if(!flag) m_bg = nullptr;
}
void MediaData::print() const {
//LOGI("MediaData", "print MediaData %d %d %d", size, size1, size2);
}

View File

@@ -0,0 +1,43 @@
#ifndef GPLAYER_MEDIADATA_H
#define GPLAYER_MEDIADATA_H
#include <string>
#include "jmat.h"
class MediaData {
private:
bool m_ref = false;
public:
MediaData();
MediaData(JMat* mat,bool ref = true);
MediaData(JMat* fmat,JMat* mmat,JMat* bmat,bool ref = true);
~MediaData();
void print() const;
int copy(MediaData* mediadata);
void reset(int flag = 0);
public:
JMat* m_data = nullptr;
JMat* m_msk = nullptr;
JMat* m_bg = nullptr;
uint8_t *data = nullptr;
uint32_t size = 0;
uint8_t *data1 = nullptr;
uint32_t size1 = 0;
uint8_t *data2 = nullptr;
uint32_t size2 = 0;
uint64_t pts = 0;
uint64_t dts = 0;
uint32_t width = 0;
uint32_t height = 0;
uint8_t flag = 0;//与FLAG_KEY_相关
};
#endif //GPLAYER_MEDIADATA_H

View File

@@ -0,0 +1,30 @@
#include <Log.h>
#include <thread>
#include "MessageHelper.h"
MessageHelper::MessageHelper(MessageSource *messageSource, MessageCb* obj) {
this->messageSource = messageSource;
msgobj = obj;
}
MessageHelper::~MessageHelper() {
notifyObj(MSG_DOMAIN_STATE, STATE_END, 0, nullptr, nullptr);
}
void MessageHelper::handleErrorMessage(Message *message) {
const char *errorMsg = error2String(message->type, (int)message->extra);
printf("===handle msg %s\n",errorMsg);
}
const char *MessageHelper::error2String(int errorCode, int errorExtra) {
const char *errorMsg = nullptr;
if (errorCode == MSG_ERROR_DEMUXING || errorCode == MSG_ERROR_DECODING) {
errorMsg = "unknown error";
}
return errorMsg;
}
void MessageHelper::notifyObj(int msgId, int arg1, long arg2, const char *msg1, const char *msg2) {
if(msgobj) msgobj->onMessageCallback(msgId, arg1, arg2, msg1, msg2);
}

Some files were not shown because too many files have changed in this diff Show More