diff --git a/app/libs/tengine-kit-sdk1.0.1.aar b/app/libs/tengine-kit-sdk1.0.1.aar deleted file mode 100644 index f22157e..0000000 Binary files a/app/libs/tengine-kit-sdk1.0.1.aar and /dev/null differ diff --git a/app/libs/tenginekit-body.aar b/app/libs/tenginekit-body.aar new file mode 100644 index 0000000..c30e50e Binary files /dev/null and b/app/libs/tenginekit-body.aar differ diff --git a/app/src/main/assets/model/tenginekit_body_landmark.tmfile b/app/libs/tenginekit-face.aar similarity index 51% rename from app/src/main/assets/model/tenginekit_body_landmark.tmfile rename to app/libs/tenginekit-face.aar index 56be3f5..624d7f9 100644 Binary files a/app/src/main/assets/model/tenginekit_body_landmark.tmfile and b/app/libs/tenginekit-face.aar differ diff --git a/app/libs/tenginekit-hand.aar b/app/libs/tenginekit-hand.aar new file mode 100644 index 0000000..7a3a718 Binary files /dev/null and b/app/libs/tenginekit-hand.aar differ diff --git a/app/libs/tenginekit.aar b/app/libs/tenginekit.aar new file mode 100644 index 0000000..842ca96 Binary files /dev/null and b/app/libs/tenginekit.aar differ diff --git a/app/src/main/assets/model/tenginekit_body_detect.tmfile b/app/src/main/assets/model/tenginekit_body_detect.tmfile deleted file mode 100644 index 9ff703a..0000000 Binary files a/app/src/main/assets/model/tenginekit_body_detect.tmfile and /dev/null differ diff --git a/app/src/main/assets/model/tenginekit_body_detect_yf.tmfile b/app/src/main/assets/model/tenginekit_body_detect_yf.tmfile deleted file mode 100644 index 0825ecd..0000000 Binary files a/app/src/main/assets/model/tenginekit_body_detect_yf.tmfile and /dev/null differ diff --git a/app/src/main/assets/model/tenginekit_face_attribute.tmfile b/app/src/main/assets/model/tenginekit_face_attribute.tmfile deleted file mode 100644 index 3b81164..0000000 Binary files a/app/src/main/assets/model/tenginekit_face_attribute.tmfile and /dev/null differ diff --git a/app/src/main/assets/model/tenginekit_face_attribute_if.tmfile b/app/src/main/assets/model/tenginekit_face_attribute_if.tmfile deleted file mode 100644 index 823002c..0000000 Binary files a/app/src/main/assets/model/tenginekit_face_attribute_if.tmfile and /dev/null differ diff --git a/app/src/main/assets/model/tenginekit_face_detect.tmfile b/app/src/main/assets/model/tenginekit_face_detect.tmfile deleted file mode 100644 index ad74f90..0000000 Binary files a/app/src/main/assets/model/tenginekit_face_detect.tmfile and /dev/null differ diff --git a/app/src/main/assets/model/tenginekit_face_iris.tmfile b/app/src/main/assets/model/tenginekit_face_iris.tmfile deleted file mode 100644 index d51d871..0000000 Binary files a/app/src/main/assets/model/tenginekit_face_iris.tmfile and /dev/null differ diff --git a/app/src/main/assets/model/tenginekit_face_landmark_2d.tmfile b/app/src/main/assets/model/tenginekit_face_landmark_2d.tmfile deleted file mode 100644 index a0e3d71..0000000 Binary files a/app/src/main/assets/model/tenginekit_face_landmark_2d.tmfile and /dev/null differ diff --git a/app/src/main/assets/model/tenginekit_insightface_scrfd2.5g_bnkps.tmfile b/app/src/main/assets/model/tenginekit_insightface_scrfd2.5g_bnkps.tmfile deleted file mode 100644 index ded8d25..0000000 Binary files a/app/src/main/assets/model/tenginekit_insightface_scrfd2.5g_bnkps.tmfile and /dev/null differ diff --git a/app/src/main/assets/model/tenginekit_seg_human_portrait.tmfile b/app/src/main/assets/model/tenginekit_seg_human_portrait.tmfile deleted file mode 100644 index 522063d..0000000 Binary files a/app/src/main/assets/model/tenginekit_seg_human_portrait.tmfile and /dev/null differ diff --git a/app/src/main/java/com/aispeech/nativedemo/face/FaceManager.java b/app/src/main/java/com/aispeech/nativedemo/face/FaceManager.java index ce84978..3225aee 100644 --- a/app/src/main/java/com/aispeech/nativedemo/face/FaceManager.java +++ b/app/src/main/java/com/aispeech/nativedemo/face/FaceManager.java @@ -23,7 +23,6 @@ import com.aispeech.nativedemo.entity.PersonInfo; import com.aispeech.nativedemo.entity.Stranger; import com.aispeech.nativedemo.log.Logger; import com.aispeech.nativedemo.network.ws.MessageUtils; -import com.aispeech.nativedemo.tengine.TEngineFaceCheckManager; import com.aispeech.nativedemo.utils.CommandExecution; import com.aispeech.nativedemo.utils.HttpUtil; import com.aispeech.nativedemo.widget.CameraTextureView; @@ -37,7 +36,6 @@ import com.shareopen.library.helper.StatusUtils; import org.json.JSONObject; import org.json.JSONTokener; -import java.io.IOException; import java.lang.ref.WeakReference; import java.text.SimpleDateFormat; import java.util.ArrayList; diff --git a/app/src/main/java/com/aispeech/nativedemo/tengine/ImageUtils.java b/app/src/main/java/com/aispeech/nativedemo/tengine/ImageUtils.java index e95eef7..f9f6f71 100644 --- a/app/src/main/java/com/aispeech/nativedemo/tengine/ImageUtils.java +++ b/app/src/main/java/com/aispeech/nativedemo/tengine/ImageUtils.java @@ -3,7 +3,7 @@ package com.aispeech.nativedemo.tengine; import static android.media.ExifInterface.ORIENTATION_ROTATE_270; import static android.media.ExifInterface.ORIENTATION_ROTATE_90; -import static com.aispeech.nativedemo.tengine.TEngineFaceCheckManager.LOG_TAG; +import static com.aispeech.nativedemo.tengine.TEngine2FaceCheckManager.LOG_TAG; import android.content.Context; import android.graphics.Bitmap; diff --git a/app/src/main/java/com/aispeech/nativedemo/tengine/TEngineFaceCheckManager.kt b/app/src/main/java/com/aispeech/nativedemo/tengine/TEngineFaceCheckManager.kt index 6caa767..09126e9 100644 --- a/app/src/main/java/com/aispeech/nativedemo/tengine/TEngineFaceCheckManager.kt +++ b/app/src/main/java/com/aispeech/nativedemo/tengine/TEngineFaceCheckManager.kt @@ -2,6 +2,7 @@ package com.aispeech.nativedemo.tengine import android.app.Activity import android.graphics.Bitmap +import android.graphics.Rect import android.util.Log import com.aispeech.nativedemo.MainActivity import com.aispeech.nativedemo.asr.observer.DuiMessageObserver @@ -13,13 +14,16 @@ import com.aispeech.nativedemo.log.Logger import com.aispeech.nativedemo.utils.Utils import com.lenovo.lefacecamerademo.ConfigClass.PROGRAM_ROOT_PATH import com.lenovo.lefacesdk.MultiAtt +import com.shareopen.library.MVVM.application.AppContext import com.shareopen.library.helper.LogUtils -import com.tenginekit.engine.core.ImageConfig -import com.tenginekit.engine.core.SdkConfig -import com.tenginekit.engine.core.TengineKitSdk -import com.tenginekit.engine.face.Face -import com.tenginekit.engine.face.FaceConfig +import com.tenginekit.AndroidConfig +import com.tenginekit.KitCore +import com.tenginekit.face.Face +import com.tenginekit.face.FaceDetectInfo +import com.tenginekit.face.FaceLandmarkInfo +import com.tenginekit.model.TenginekitPoint import java.io.File +import java.nio.ByteBuffer import java.util.concurrent.ArrayBlockingQueue import java.util.concurrent.ConcurrentLinkedQueue @@ -28,7 +32,7 @@ class TEngineFaceCheckManager : FaceCheckInterface { companion object { val instance by lazy (LazyThreadSafetyMode.SYNCHRONIZED){ - TEngineFaceCheckManager() + TEngine2FaceCheckManager() } const val LOG_TAG = "TengineKitApp" } @@ -39,12 +43,12 @@ class TEngineFaceCheckManager : FaceCheckInterface { @Volatile var faceCount = 0 var faceQueue = ArrayBlockingQueue(10) - var faceListInVad = ConcurrentLinkedQueue() + var faceListInVad = ConcurrentLinkedQueue() var faceStrListInVad = ConcurrentLinkedQueue() var faceRectListInVad = ConcurrentLinkedQueue() - var faceListCloseInVad = ConcurrentLinkedQueue() + var faceListCloseInVad = ConcurrentLinkedQueue() var faceStrListCloseInVad = ConcurrentLinkedQueue() var faceRectListCloseInVad = ConcurrentLinkedQueue() @@ -64,7 +68,18 @@ class TEngineFaceCheckManager : FaceCheckInterface { override fun init() { Log.i(LOG_TAG, "start init") - copyModel(MainActivity.instance) + /** + * 初始化 + * */ + KitCore.init( + AppContext.getInstance(), + AndroidConfig + .create() + .setNormalMode() + .setDefaultFunc() + .setInputImageFormat(AndroidConfig.ImageFormat.RGBA)) +// .setInputImageSize(Image_w, Image_h) +// .setOutputImageSize(Image_w as Int, Image_h as Int) Log.i(LOG_TAG, "init Success") } @@ -77,11 +92,11 @@ class TEngineFaceCheckManager : FaceCheckInterface { override fun copyFinish() { Log.i(LOG_TAG, "copyModel Success") //modelCopyFinished = true - val sdkConfig = SdkConfig() - sdkConfig.setAllowReport(false) - //TengineKitSdk.getInstance().initSdk(PROGRAM_ROOT_PATH, sdkConfig, AppContext.getInstance()) - TengineKitSdk.getInstance().initSdk(activity.externalCacheDir!!.absolutePath, sdkConfig, activity) - TengineKitSdk.getInstance().initFaceDetect() +// val sdkConfig = SdkConfig() +// sdkConfig.setAllowReport(false) +// //TengineKitSdk.getInstance().initSdk(PROGRAM_ROOT_PATH, sdkConfig, AppContext.getInstance()) +// TengineKitSdk.getInstance().initSdk(activity.externalCacheDir!!.absolutePath, sdkConfig, activity) +// TengineKitSdk.getInstance().initFaceDetect() } override fun copyFail() { @@ -179,53 +194,55 @@ class TEngineFaceCheckManager : FaceCheckInterface { return null } - fun doDetect(bitmap: Bitmap, personInfo: PersonInfo) : Face? { + fun doDetect(bitmap: Bitmap, personInfo: PersonInfo) : FaceLandmarkInfo? { val start = System.currentTimeMillis() - var faceResult : Face? = null; + var faceResult : FaceLandmarkInfo? = null; bitmap?.let { - val byte = ImageUtils.bitmap2RGB(bitmap) - val config = FaceConfig().apply { - detect = true - landmark2d = true + /** + * 获取人脸信息 + */ + /** + * 获取人脸信息 + */ + val data: ByteArray = bitmap2Bytes(bitmap) + val faceDetect = Face.detect(data) + var faceDetectInfos: List? = ArrayList() + var landmarkInfos: List = ArrayList() + if (faceDetect.faceCount > 0) { + faceDetectInfos = faceDetect.detectInfos + landmarkInfos = faceDetect.landmark2d() } - val imageConfig = ImageConfig().apply { - data = byte - degree = 0 - mirror = false - height = it.height - width = it.width - format = ImageConfig.FaceImageFormat.RGB - } - val faces = TengineKitSdk.getInstance().detectFace(imageConfig, config) - if (faces.isNotEmpty()) { - Log.i(LOG_TAG, "faces length:" + faces.size + " duration " + (System.currentTimeMillis() - start)) + Log.d("#####", "Face Num: " + faceDetectInfos!!.size) + if (faceDetectInfos != null && faceDetectInfos.size > 0) { + val face_landmarks: List> = ArrayList() + for (i in faceDetectInfos.indices) { + var rect: Rect? = Rect() + rect = faceDetectInfos[i].asRect() + Log.d( + "#####", + "mouseClose: " + landmarkInfos[i].mouseClose + "pitch: " + landmarkInfos[i].pitch + "yaw: " + landmarkInfos[i].yaw + ) + } } Log.i(LOG_TAG, "end detect") val sb = StringBuilder() - if (faces != null) { - LogUtils.e("access123", "faces length " + faces.size) - for (face in faces) { - sb.append("· ").append("性别：").append(if (face.gender == 1) "女" else "男") - .append("\n") - sb.append("· ").append("年龄：").append(face.age).append("\n") -// sb.append("· ").append("模糊度：").append(df.format(face.blurness.toDouble())) -// .append("\n") -// sb.append("· ").append("置信度：").append(df.format(face.confidence.toDouble())) -// .append("\n") + if (landmarkInfos != null) { + LogUtils.e("access123", "faces length " + landmarkInfos.size) + for (face in landmarkInfos) { var expression = "" sb.append("· ").append("表情：").append(expression).append("\n") sb.append("· ").append("眼部状态(左眼)：").append(face.leftEyeClose) .append("\n") sb.append("· ").append("眼部状态(右眼)：").append(face.rightEyeClose) .append("\n") - sb.append("· ").append("嘴部状态：").append("闭嘴${(face.mouthClose*100).toInt()}%") + sb.append("· ").append("嘴部状态：").append("闭嘴${(face.mouseClose*100).toInt()}%") .append("\n") - sb.append("· ").append("嘴部状态bigopen：").append("闭嘴${(face.mouthBigOpen*100).toInt()}%") + sb.append("· ").append("嘴部状态bigopen：").append("闭嘴${(face.mouseOpenBig*100).toInt()}%") .append("\n") - sb.append("· ").append("3DPose：{pitch=").append(face.headX).append(",roll=") - .append(face.headZ).append(",yaw=").append(face.headY).append("}\n") - sb.append("· ").append("关键点个数：").append("" + face.landmark.size).append("\n") + sb.append("· ").append("3DPose：{pitch=").append(face.pitch).append(",roll=") + .append(face.roll).append(",yaw=").append(face.yaw).append("}\n") + sb.append("· ").append("关键点个数：").append("" + face.landmarks.size).append("\n") sb.append("\n") Log.e(LOG_TAG, "bitmapDetected: $sb") faceResult = face @@ -242,6 +259,14 @@ class TEngineFaceCheckManager : FaceCheckInterface { return faceResult; } + private fun bitmap2Bytes(image: Bitmap): ByteArray { + // calculate how many bytes our image consists of + val bytes = image.byteCount + val buffer = ByteBuffer.allocate(bytes) // Create a new buffer + image.copyPixelsToBuffer(buffer) // Move the byte data to the buffer + return buffer.array() + } + private fun getEyeStatus(eyeType: Int): String? { var reuslt = "" when (eyeType) { @@ -266,14 +291,14 @@ class TEngineFaceCheckManager : FaceCheckInterface { return result } - fun addFaceInVad(faceResult: Face?, faceStr: String?, faceMultiAtt: MultiAtt?) { + fun addFaceInVad(faceResult: FaceLandmarkInfo?, faceStr: String?, faceMultiAtt: MultiAtt?) { faceListInVad.offer(faceResult) faceStrListInVad.offer(faceStr) faceRectListInVad.offer(faceMultiAtt) } fun addFaceInVadClose( - faceResult: Face?, + faceResult: FaceLandmarkInfo?, faceStr: String?, faceMultiAtt: MultiAtt? ) { @@ -306,7 +331,9 @@ class TEngineFaceCheckManager : FaceCheckInterface { return checkSpeak(name, " BeforeASR ", faceListInVad) } - override fun onDestroy() {} + override fun onDestroy() { + KitCore.release() + } override fun getCurrentSpeak(name: String, isClear: Boolean): Boolean { @@ -415,7 +442,7 @@ class TEngineFaceCheckManager : FaceCheckInterface { private fun checkAngle( - faceList: ConcurrentLinkedQueue, + faceList: ConcurrentLinkedQueue, name: String ): Boolean { val count = faceList.size @@ -425,12 +452,12 @@ class TEngineFaceCheckManager : FaceCheckInterface { overAngleCount++ continue } - if (face.headY > 0.3f || face.headY < -0.3f) { + if (face.yaw > 0.3f || face.yaw < -0.3f) { overAngleCount++ } else { LogUtils.e( "testtwoface", - "face.yaw " + face.headY + " time " + DuiMessageObserver.mTime + " " + name + "face.yaw " + face.yaw + " time " + DuiMessageObserver.mTime + " " + name ) } } @@ -449,7 +476,7 @@ class TEngineFaceCheckManager : FaceCheckInterface { fun checkSpeak( name: String, logTag: String, - faceList: ConcurrentLinkedQueue + faceList: ConcurrentLinkedQueue ): Boolean { val start = System.currentTimeMillis() var isSpeak = false @@ -464,16 +491,16 @@ class TEngineFaceCheckManager : FaceCheckInterface { if (face == null) { continue } - if (faceType >= 0 && (faceType - face.mouthClose) > 0.1f) { + if (faceType >= 0 && (faceType - face.mouseClose) > 0.1f) { mouthChangeCount++ } - faceType = face.mouthClose - if (face.mouthClose < 0.9f) { + faceType = face.mouseClose + if (face.mouseClose < 0.9f) { mouthOpen++ } else { mouthClose++ } - if (face.headX > 0.3f || face.headX < -0.3f) { + if (face.pitch > 0.3f || face.pitch < -0.3f) { overAngleCount++ } } diff --git a/app/src/main/java/com/aispeech/nativedemo/tengine/copyAssetFolder.kt b/app/src/main/java/com/aispeech/nativedemo/tengine/copyAssetFolder.kt index fb7374b..23d334e 100644 --- a/app/src/main/java/com/aispeech/nativedemo/tengine/copyAssetFolder.kt +++ b/app/src/main/java/com/aispeech/nativedemo/tengine/copyAssetFolder.kt @@ -1,8 +1,6 @@ package com.aispeech.nativedemo.tengine import android.content.res.AssetManager -import com.aispeech.nativedemo.tengine.ImageUtils.isFileExists -import com.shareopen.library.helper.util.FileUtils import java.io.File import java.io.File.separator import java.io.FileOutputStream diff --git a/app/src/main/java/com/aispeech/nativedemo/tengine2/TEngine2FaceCheckManager.kt b/app/src/main/java/com/aispeech/nativedemo/tengine2/TEngine2FaceCheckManager.kt new file mode 100644 index 0000000..c768798 --- /dev/null +++ b/app/src/main/java/com/aispeech/nativedemo/tengine2/TEngine2FaceCheckManager.kt @@ -0,0 +1,511 @@ +package com.aispeech.nativedemo.tengine2 + +import android.app.Activity +import android.graphics.Bitmap +import android.graphics.Rect +import android.util.Log +import com.aispeech.nativedemo.MainActivity +import com.aispeech.nativedemo.asr.observer.DuiMessageObserver +import com.aispeech.nativedemo.entity.PersonInfo +import com.aispeech.nativedemo.face.AngleInfo +import com.aispeech.nativedemo.face.FaceCheckInterface +import com.aispeech.nativedemo.face.FaceManager +import com.aispeech.nativedemo.log.Logger +import com.aispeech.nativedemo.utils.Utils +import com.lenovo.lefacecamerademo.ConfigClass.PROGRAM_ROOT_PATH +import com.lenovo.lefacesdk.MultiAtt +import com.shareopen.library.MVVM.application.AppContext +import com.shareopen.library.helper.LogUtils +import com.tenginekit.AndroidConfig +import com.tenginekit.KitCore +import com.tenginekit.face.Face +import com.tenginekit.face.FaceDetectInfo +import com.tenginekit.face.FaceLandmarkInfo +import com.tenginekit.model.TenginekitPoint +import java.io.File +import java.nio.ByteBuffer +import java.util.concurrent.ArrayBlockingQueue +import java.util.concurrent.ConcurrentLinkedQueue + + +class TEngine2FaceCheckManager : FaceCheckInterface { + + companion object { + val instance by lazy (LazyThreadSafetyMode.SYNCHRONIZED){ + TEngine2FaceCheckManager() + } + const val LOG_TAG = "TengineKitApp" + } + var hasInitKs = false + private val TAG = "KSFaceCheckManager" + var needKSCheck = false + + @Volatile + var faceCount = 0 + var faceQueue = ArrayBlockingQueue(10) + var faceListInVad = ConcurrentLinkedQueue() + var faceStrListInVad = ConcurrentLinkedQueue() + var faceRectListInVad = ConcurrentLinkedQueue() + + + var faceListCloseInVad = ConcurrentLinkedQueue() + var faceStrListCloseInVad = ConcurrentLinkedQueue() + var faceRectListCloseInVad = ConcurrentLinkedQueue() + + var needFaceCheck = true + var needFaceAngleCheck = true + +// fun getInstance(): TEngineFaceCheckManager? { +// if (TEngineFaceCheckManager.mInstance == null) { +// synchronized(KSFaceCheckManager::class.java) { +// if (TEngineFaceCheckManager.mInstance == null) { +// TEngineFaceCheckManager.mInstance = TEngineFaceCheckManager() +// } +// } +// } +// return TEngineFaceCheckManager.mInstance +// } + + override fun init() { + Log.i(LOG_TAG, "start init") + /** + * 初始化 + * */ + KitCore.init( + AppContext.getInstance(), + AndroidConfig + .create() + .setNormalMode() + .setDefaultFunc() + .setInputImageFormat(AndroidConfig.ImageFormat.RGBA)) +// .setInputImageSize(Image_w, Image_h) +// .setOutputImageSize(Image_w as Int, Image_h as Int) + Log.i(LOG_TAG, "init Success") + + } + + override fun bitmapDetected(bitmap: Bitmap?, personInfo: PersonInfo) { + if (!needKSCheck || !hasInitKs) { + return + } + if (personInfo.result == null || DuiMessageObserver.needPause) { + return + } + val start = System.currentTimeMillis() + val bitmap1 = Utils.CompressBmpByHD(bitmap, personInfo.result) + ?: return + personInfo.faceStr = "" + val faceResult = doDetect(bitmap1, personInfo) + if (faceResult != null) { + addFaceInVad(faceResult, personInfo.faceStr, personInfo.result) + } else { + LogUtils.e( + "testface", + " faceData.faceResult == null $personInfo" + ) + } + // faceQueue.offer(personInfo); +// faceCount++; + Log.e( + "testtwo", + "selectMan CompressBmpByHD duration " + (System.currentTimeMillis() - start) + ) + if (MainActivity.instance != null) { + MainActivity.instance.setSettingLog(personInfo.faceStr) + } + } + + override fun bitmapDetectedClose(bitmap: Bitmap?, personInfo: PersonInfo) { + if (!needKSCheck || !hasInitKs) { + return + } + if (personInfo.result == null || DuiMessageObserver.needPause) { + LogUtils.e("testtwo", " personInfo.result == null$personInfo") + return + } + val start = System.currentTimeMillis() + val bitmap1 = Utils.CompressBmpByHD(bitmap, personInfo.result) + ?: return + personInfo.faceStr = "" + val faceResult = doDetect(bitmap1, personInfo) + val time = System.currentTimeMillis() + if (faceResult != null) { + addFaceInVadClose(faceResult, personInfo.faceStr, personInfo.result) + } else { + LogUtils.e( + "testtwo", + " faceData.faceResult == null $personInfo" + ) + val file: File = + File(PROGRAM_ROOT_PATH + "/" + DuiMessageObserver.mTime + "_" + personInfo.faceWidth + "_" + time + "_" + "last.jpg") + //BitmapUtil.saveBitmap(file.getAbsolutePath(), bitmap1); + } + // faceQueue.offer(personInfo); +// faceCount++; + Log.e("testtwo", "close man face width " + personInfo.faceWidth + " time " + time) + Log.e( + "testtwo", + "close CompressBmpByHD duration " + (System.currentTimeMillis() - start) + " facestr " + personInfo.faceStr + " person " + personInfo + " time " + time + ) + if (MainActivity.instance != null) { + MainActivity.instance.setSettingLogClose(personInfo.faceStr) + } + } + + override fun onVadBegin() { + while (faceListInVad.size > 2) { + faceListInVad.poll() + } + Log.e("testface", "before vad " + faceListInVad.size) + } + + override fun setFaceCheck(faceCheck: Boolean, + needMouthCheck: Boolean, + needAngleCheck: Boolean) { + needKSCheck = faceCheck + needFaceCheck = needMouthCheck + needFaceAngleCheck = needAngleCheck; + + Log.i(LOG_TAG, "setFaceCheck ") + } + + override fun getAngle(bitmap: Bitmap?, personInfo: PersonInfo?): AngleInfo? { + return null + } + + fun doDetect(bitmap: Bitmap, personInfo: PersonInfo) : FaceLandmarkInfo? { + val start = System.currentTimeMillis() + var faceResult : FaceLandmarkInfo? = null; + bitmap?.let { + /** + * 获取人脸信息 + */ + /** + * 获取人脸信息 + */ + val data: ByteArray = bitmap2Bytes(bitmap) + val faceDetect = Face.detect(data) + var faceDetectInfos: List? = ArrayList() + var landmarkInfos: List = ArrayList() + if (faceDetect.faceCount > 0) { + faceDetectInfos = faceDetect.detectInfos + landmarkInfos = faceDetect.landmark2d() + } + + Log.d("#####", "Face Num: " + faceDetectInfos!!.size) + if (faceDetectInfos != null && faceDetectInfos.size > 0) { + val face_landmarks: List> = ArrayList() + for (i in faceDetectInfos.indices) { + var rect: Rect? = Rect() + rect = faceDetectInfos[i].asRect() + Log.d( + "#####", + "mouseClose: " + landmarkInfos[i].mouseClose + "pitch: " + landmarkInfos[i].pitch + "yaw: " + landmarkInfos[i].yaw + ) + } + } + Log.i(LOG_TAG, "end detect") + val sb = StringBuilder() + if (landmarkInfos != null) { + LogUtils.e("access123", "faces length " + landmarkInfos.size) + for (face in landmarkInfos) { + var expression = "" + sb.append("· ").append("表情：").append(expression).append("\n") + sb.append("· ").append("眼部状态(左眼)：").append(face.leftEyeClose) + .append("\n") + sb.append("· ").append("眼部状态(右眼)：").append(face.rightEyeClose) + .append("\n") + sb.append("· ").append("嘴部状态：").append("闭嘴${(face.mouseClose*100).toInt()}%") + .append("\n") + sb.append("· ").append("嘴部状态bigopen：").append("闭嘴${(face.mouseOpenBig*100).toInt()}%") + .append("\n") + sb.append("· ").append("3DPose：{pitch=").append(face.pitch).append(",roll=") + .append(face.roll).append(",yaw=").append(face.yaw).append("}\n") + sb.append("· ").append("关键点个数：").append("" + face.landmarks.size).append("\n") + sb.append("\n") + Log.e(LOG_TAG, "bitmapDetected: $sb") + faceResult = face + } + } + val faceStr = sb.toString() + personInfo.faceStr = faceStr + LogUtils.e("access123", "faceStr $faceStr") + LogUtils.e( + "testface", + "face duration " + (System.currentTimeMillis() - start) + " lx x1 " + personInfo.result.fa_x1 + " lx y1 " + personInfo.result.fa_y1 + " lx w " + personInfo.result.fa_w + " lx h " + personInfo.result.fa_h + ) + } + return faceResult; + } + + private fun bitmap2Bytes(image: Bitmap): ByteArray { + // calculate how many bytes our image consists of + val bytes = image.byteCount + val buffer = ByteBuffer.allocate(bytes) // Create a new buffer + image.copyPixelsToBuffer(buffer) // Move the byte data to the buffer + return buffer.array() + } + + private fun getEyeStatus(eyeType: Int): String? { + var reuslt = "" + when (eyeType) { + 0 -> reuslt = "不戴眼镜，并且睁着眼" + 1 -> reuslt = "不戴眼镜，并且闭着眼" + 2 -> reuslt = "戴着普通眼镜，并且睁着眼" + 3 -> reuslt = "戴着普通眼镜，并且闭着眼" + 4 -> reuslt = "戴着墨镜" + 5 -> reuslt = "眼镜被遮挡" + } + return reuslt + } + + private fun getMouthStatus(mouthStatus: Int): String? { + var result = "" + when (mouthStatus) { + 0 -> result = "带着面具或者带着口罩" + 1 -> result = "被其他东西遮挡着嘴巴" + 2 -> result = "闭嘴状态" + 3 -> result = "张嘴状态" + } + return result + } + + fun addFaceInVad(faceResult: FaceLandmarkInfo?, faceStr: String?, faceMultiAtt: MultiAtt?) { + faceListInVad.offer(faceResult) + faceStrListInVad.offer(faceStr) + faceRectListInVad.offer(faceMultiAtt) + } + + fun addFaceInVadClose( + faceResult: FaceLandmarkInfo?, + faceStr: String?, + faceMultiAtt: MultiAtt? + ) { + faceListCloseInVad.offer(faceResult) + faceStrListCloseInVad.offer(faceStr) + faceRectListCloseInVad.offer(faceMultiAtt) + } + + override fun getCurrentSpeakBeforeASR(name: String): Boolean { + if (!needKSCheck || !hasInitKs) { + return true + } + if (!needFaceCheck) { + return if (!needFaceAngleCheck) { + true + } else { + checkAngle(faceListInVad, name) + } + } + if (faceListInVad.size == 0) { + Log.e("testface", "BeforeASR size 0 is speak false") + return false + } + if (needFaceAngleCheck) { + val isAngleOk = checkAngle(faceListInVad, name) + if (!isAngleOk) { + return false + } + } + return checkSpeak(name, " BeforeASR ", faceListInVad) + } + + override fun onDestroy() { + KitCore.release() + } + + + override fun getCurrentSpeak(name: String, isClear: Boolean): Boolean { + Log.e("testkd2", "FaceManager.needKSCheck " + needKSCheck) + if (!needKSCheck || !hasInitKs) { + return true + } + if (!needFaceCheck) { + return if (!needFaceAngleCheck) { + true + } else { + val isRightAngle = checkAngle(faceListInVad, name) + faceListInVad.clear() + faceStrListInVad.clear() + faceRectListInVad.clear() + isRightAngle + } + } + if (faceListInVad.size == 0) { + Log.e("testface2", "close size 0 is speak false") + return false + } + if (needFaceAngleCheck) { + val isAngleOk = checkAngle(faceListInVad, name) + Log.e("testtwoface", "$name isAngleOk $isAngleOk") + if (!isAngleOk) { + faceListInVad.clear() + faceStrListInVad.clear() + faceRectListInVad.clear() + return false + } + } + //needPause = true; + val start = System.currentTimeMillis() + //if (FaceManager.faceCount == 1 && faceListInVad.size() < 20) + run {} + // while (FaceManager.faceCount != 0) { +// try { +// Thread.sleep(20); +// } catch (InterruptedException e) { +// throw new RuntimeException(e); +// } +// } + val isSpeak = checkSpeak(name, "closeMan", faceListInVad) + if (!isSpeak) { + } + faceListInVad.clear() + faceStrListInVad.clear() + faceRectListInVad.clear() + return isSpeak + } + + override fun getCurrentCloseSpeak(closeName: String, isClear: Boolean): Boolean { + if (!FaceManager.needKSCheck || !hasInitKs) { + return true + } + if (!needFaceCheck) { + return if (!needFaceAngleCheck) { + true + } else { + val isRightAngle = checkAngle( + faceListCloseInVad, + "closeName-$closeName" + ) + faceListCloseInVad.clear() + faceStrListCloseInVad.clear() + faceRectListCloseInVad.clear() + isRightAngle + } + } + if (faceListCloseInVad.size == 0) { + Log.e("testface", "size 0 is speak false") + return false + } + if (needFaceAngleCheck) { + val isAngleOk = checkAngle( + faceListCloseInVad, + "closeName-$closeName" + ) + if (!isAngleOk) { + faceListCloseInVad.clear() + faceStrListCloseInVad.clear() + faceRectListCloseInVad.clear() + return false + } + } + //needPause = true; + val start = System.currentTimeMillis() + //if (FaceManager.faceCount == 1 && faceListInVad.size() < 20) + run {} + // while (FaceManager.faceCount != 0) { +// try { +// Thread.sleep(20); +// } catch (InterruptedException e) { +// throw new RuntimeException(e); +// } +// } + val isSpeak = checkSpeak(closeName, "", faceListCloseInVad) + if (!isSpeak) { + } + faceListCloseInVad.clear() + faceStrListCloseInVad.clear() + faceRectListCloseInVad.clear() + return isSpeak + } + + + private fun checkAngle( + faceList: ConcurrentLinkedQueue, + name: String + ): Boolean { + val count = faceList.size + var overAngleCount = 0 + for (face in faceList) { + if (face == null) { + overAngleCount++ + continue + } + if (face.yaw > 0.3f || face.yaw < -0.3f) { + overAngleCount++ + } else { + LogUtils.e( + "testtwoface", + "face.yaw " + face.yaw + " time " + DuiMessageObserver.mTime + " " + name + ) + } + } + val rightAngleCount = count - overAngleCount + LogUtils.e( + "testtwoface", + "face.yaw overAngleCount $overAngleCount count $count name $name" + ) + return if (count <= 12) { + rightAngleCount * 10 > count + } else { + rightAngleCount * 3 > count + } + } + + fun checkSpeak( + name: String, + logTag: String, + faceList: ConcurrentLinkedQueue + ): Boolean { + val start = System.currentTimeMillis() + var isSpeak = false + var mouthOpen = 0 + var mouthClose = 0 + var mouthOther = 0 + var overAngleCount = 0 + var faceType = -1f + var mouthChangeCount = 0 + val count = faceList.size + for (face in faceList) { + if (face == null) { + continue + } + if (faceType >= 0 && (faceType - face.mouseClose) > 0.1f) { + mouthChangeCount++ + } + faceType = face.mouseClose + if (face.mouseClose < 0.9f) { + mouthOpen++ + } else { + mouthClose++ + } + if (face.pitch > 0.3f || face.pitch < -0.3f) { + overAngleCount++ + } + } + if (faceList.size <= 8) { + isSpeak = mouthChangeCount > 0 || overAngleCount > 0 + } else if (overAngleCount > 0) { + isSpeak = mouthChangeCount > 0 || overAngleCount * 16 >= count + } else if (mouthOpen > 0 && mouthClose > 0 || mouthOpen > 0 && mouthOther > 0) { + // || (mouthClose > 0 && mouthOther > 0) + isSpeak = true + } + if (!isSpeak) { + if (mouthChangeCount * 2 >= count) { + isSpeak = true + } else if (mouthOpen == 0 && count <= 20) { + if (Math.abs(mouthClose - mouthOther) < 3 || mouthChangeCount > mouthClose || mouthChangeCount > mouthOther) { + isSpeak = true + } + } + } + Logger.e("djASR-嘴型检测: " + logTag + name + " is speak " + isSpeak + " mouthOpen " + mouthOpen + " mouthClose " + mouthClose + " mouthOther " + mouthOther + " mouthChangeCount " + mouthChangeCount + " leftFaceSize " + faceCount + " overAngleCount " + overAngleCount + " time " + DuiMessageObserver.mTime) + Log.e( + "testface", + logTag + name + " is speak " + isSpeak + " mouthOpen " + mouthOpen + " mouthClose " + mouthClose + " mouthOther " + mouthOther + " mouthChangeCount " + mouthChangeCount + " leftFaceSize " + faceCount + " overAngleCount " + overAngleCount + " time " + DuiMessageObserver.mTime + " duration " + (System.currentTimeMillis() - start) + ) + return isSpeak + } + +}