AR 眼镜之-条形码识别-实现方案

目录

📂 前言

AR 眼镜系统版本

条形码识别

1. 🔱 技术方案

1.1 技术方案概述

1.2 实现方案

1）相机App显示模块

2）算法so库JNI模块

3）算法条形码识别模块

2. 💠 实现相机App显示模块

2.1 创建 BarcodeIdentifyDemoActivity.kt

2.2 创建 activity_barcode_identify_demo.xml

2.3 创建 AlgorithmLibHelper.kt

2.4 创建 AssetsHelper.kt

2.5 创建 ProductBean.kt

3. ⚛️ 算法so库JNI模块

3.1 新建CMakeLists.txt，引入算法so库以及头文件

1）CMakeLists.txt 内容如下：

2）引入算法so库以及头文件

3.2 新建cpp文件，调用算法so库方法

3.3 新建native方法，加载JNI模块生成的库以及映射对应方法

3.4 配置 build.gradle 文件

4. ✅ 小结

---

📂 前言

AR 眼镜系统版本

        W517 Android9。

条形码识别

        AR眼镜中相机App，调用算法条形码识别接口，获取到算法接口返回文本后，将文本显示在眼镜页面。

1. 🔱 技术方案

1.1 技术方案概述

        条形码识别功能的实现，主要包括以下三大模块：相机App显示模块、算法so库JNI模块、以及算法条形码识别模块。

1.2 实现方案

1）相机App显示模块

1. 实现相机预览、拍照与保存功能；

2. 传入拍照后的图片路径，调用算法so库JNI模块；

3. 显示算法so库JNI模块返回的条形码识别到的商品信息。

2）算法so库JNI模块

1. 新建CMakeLists.txt，引入算法so库以及头文件；

2. 新建cpp文件，调用算法so库方法；

3. 新建native方法，加载JNI模块生成的库以及映射对应方法。

3）算法条形码识别模块

1. 对照片进行处理，获取到照片中的二维码；

2. 调用二维码识别so库，获取二维码中的信息；

3. 将二维码信息与数据库匹配，返回匹配到的商品信息。

2. 💠 实现相机App显示模块

2.1 创建 BarcodeIdentifyDemoActivity.kt

        主要实现相机预览、拍照与保存等功能。可参考《一周内从0到1开发一款 AR眼镜 相机应用？》

class BarcodeIdentifyDemoActivity :BaseActivity<ActivityBarcodeIdentifyDemoBinding, MainViewModel>() {private val TAG = BarcodeIdentifyDemoActivity::class.java.simpleNameprivate val DEFAULT_CONTENT ="    \"商品名称\": \"\",\n" + "    \"品牌\": \"\",\n" + "    \"规格型号\": \"\",\n" + "    \"价格\": \"\",\n" + "    \"原产地\": \"\",\n" + "    \"税率\": \"\",\n" + "    \"税号\": \"\",\n" + "    \"功能用途\":\"\""private val CAMERA_MAX_RESOLUTION = Size(3264, 2448)private var mCameraExecutor: ExecutorService = Executors.newSingleThreadExecutor()private var mImageCapture: ImageCapture? = nullprivate var mIsBarcodeRecognition: Boolean = falseprivate var mTimeOut: CountDownTimer? = nulloverride fun initBinding(inflater: LayoutInflater): ActivityBarcodeIdentifyDemoBinding =ActivityBarcodeIdentifyDemoBinding.inflate(inflater)override fun initData() {AlgorithmLibHelper.init(this)AlgorithmLibHelper.productBeanLiveData.observe(this) {Log.i(TAG, "initData: $it")if (it != null) {runOnUiThread {binding.loading.visibility = GONEmIsBarcodeRecognition = falsebinding.content.text ="    \"商品名称\": \"${it.product_name}\",\n" + "    \"品牌\": \"${it.brand}\",\n" + "    \"规格型号\": \"${it.specifications}\",\n" + "    \"价格\": \"${it.price}\",\n" + "    \"原产地\": \"${it.country_of_origin}\",\n" + "    \"税率\": \"${it.tax_rate}\",\n" + "    \"税号\": \"${it.tax_code}\",\n" + "    \"功能用途\":\"${it.function_and_use}\""mTimeOut?.cancel()mTimeOut = null}} else {errorHandle()}}}override fun initViewModel() {viewModel.init(this)}override fun initView() {initWindow()switchToPhoto()binding.parent.setOnClickListener {try {if (mImageCapture == null || mIsBarcodeRecognition) {AGGToast(this, Toast.LENGTH_SHORT, "please hold on").show()} else {mIsBarcodeRecognition = truebinding.loading.setContent("Identify...")binding.loading.visibility = VISIBLEtakePicture()}} catch (e: Exception) {e.printStackTrace()}}binding.loading.visibility = VISIBLE}override fun onResume() {Log.i(TAG, "onResume: ")super.onResume()}override fun onStop() {Log.i(TAG, "onStop: ")super.onStop()}override fun onDestroy() {Log.i(TAG, "onDestroy: ")mTimeOut?.cancel()mTimeOut = nullAnimatorSwitchHelper.isAnimating = falseAnimatorSwitchHelper.isFirstSwitch = truesuper.onDestroy()mCameraExecutor.shutdown()XrEnvironment.getInstance().imuReset()AlgorithmLibHelper.release()}private fun initWindow() {Log.i(TAG, "initWindow: ")val lp = window.attributeslp.dofIndex = 0lp.subType = WindowManager.LayoutParams.WINDOW_IMMERSIVE_0DOFwindow.attributes = lp}private fun switchToPhoto() {Log.i(TAG, "switchToPhoto: ")val cameraProviderFuture = ProcessCameraProvider.getInstance(this)cameraProviderFuture.addListener({try {mImageCapture = ImageCapture.Builder().setCaptureMode(ImageCapture.CAPTURE_MODE_MAXIMIZE_QUALITY).setTargetResolution(CAMERA_MAX_RESOLUTION).build()val cameraSelector = CameraSelector.DEFAULT_BACK_CAMERAval cameraProvider = cameraProviderFuture.get()cameraProvider.unbindAll()// 可预览val preview = Preview.Builder().build()binding.previewView.apply {implementationMode = PreviewView.ImplementationMode.COMPATIBLEpreview.setSurfaceProvider(surfaceProvider)clipToOutline = truevisibility = VISIBLE}cameraProvider.bindToLifecycle(this, cameraSelector, preview, mImageCapture)// 无预览
//                cameraProvider.bindToLifecycle(this, cameraSelector, mImageCapture)binding.loading.visibility = GONE} catch (e: java.lang.Exception) {Log.e(TAG, "bindCamera Failed！: $e")}}, ContextCompat.getMainExecutor(this))}/*** 拍照*/private fun takePicture() {Log.i(TAG, "takePicture: ")SoundPoolTools.playCameraPhoto(this)val photoFile = viewModel.createPhotoFile()mImageCapture?.takePicture(ImageCapture.OutputFileOptions.Builder(photoFile).build(),mCameraExecutor,object : ImageCapture.OnImageSavedCallback {override fun onError(exc: ImageCaptureException) {Log.e(TAG, "Photo capture failed: ${exc.message}", exc)}@SuppressLint("SetTextI18n")override fun onImageSaved(output: ImageCapture.OutputFileResults) {val savedUri = output.savedUri ?: Uri.fromFile(photoFile)Log.i(TAG, "Photo capture succeeded: ${savedUri.path}")runOnUiThread { updateFlashPreview(savedUri) }viewModel.updateMediaFile(this@BarcodeIdentifyDemoActivity, photoFile)// 调用条形码识别算法lifecycleScope.launch {withContext(Dispatchers.IO) {savedUri.path?.let {AlgorithmLibHelper.identifyBarcode(it)}}}// 超时逻辑runOnUiThread {mTimeOut?.cancel()mTimeOut = nullmTimeOut = object : CountDownTimer(15000L, 1000) {override fun onTick(millisUntilFinished: Long) {}override fun onFinish() {Log.e(TAG, "onFinish: identify timeout")AGGToast(this@BarcodeIdentifyDemoActivity,Toast.LENGTH_SHORT,"identify timeout").show()errorHandle()}}.start()}}})}private fun updateFlashPreview(savedUri: Uri) {binding.flashPreview.apply {visibility = VISIBLEGlide.with(this@BarcodeIdentifyDemoActivity).load(savedUri).into(this)// 创建动画val animatorAlpha = ObjectAnimator.ofFloat(this, "alpha", 0f, 1f)val animatorX = ObjectAnimator.ofFloat(this, "translationX", 0f, SizeUtils.dp2px(-144f).toFloat())// 同时播放X和Y轴的动画val animatorSet = AnimatorSet()animatorSet.playTogether(animatorAlpha, animatorX)animatorSet.interpolator = EaseOutInterpolator()animatorSet.duration = CAMERA_FLASH_PREVIEW_ANIM_TIMEanimatorSet.start()// 设置动画监听器，在动画结束后等待2秒，然后隐藏图片animatorSet.addListener(object : AnimatorListenerAdapter() {override fun onAnimationEnd(animation: Animator) {super.onAnimationEnd(animation)// 2秒后隐藏图片postDelayed({visibility = GONE}, CAMERA_FLASH_PREVIEW_SHOW_TIME)}})}}private fun errorHandle() {runOnUiThread {binding.content.text = DEFAULT_CONTENTbinding.loading.visibility = GONEmIsBarcodeRecognition = falsemTimeOut?.cancel()mTimeOut = null}}}

2.2 创建 activity_barcode_identify_demo.xml

        包括显示算法so库JNI模块返回的条形码识别到的商品信息。

<?xml version="1.0" encoding="utf-8"?>
<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"xmlns:app="http://schemas.android.com/apk/res-auto"android:id="@+id/parent"android:layout_width="match_parent"android:layout_height="match_parent"android:keepScreenOn="true"><com.agg.ui.AGGActionBarandroid:id="@+id/title"android:layout_width="wrap_content"android:layout_height="wrap_content"app:UIActionBarTitle="Barcode Identify"app:UITitleLevel="M"app:layout_constraintStart_toStartOf="parent"app:layout_constraintTop_toTopOf="parent" /><com.agg.ui.AGGTextViewandroid:id="@+id/content"style="@style/TextBody5"android:layout_width="match_parent"android:layout_height="0dp"android:layout_marginHorizontal="32dp"android:layout_marginTop="16dp"android:layout_marginBottom="64dp"android:gravity="center_vertical|start"app:layout_constraintBottom_toBottomOf="parent"app:layout_constraintEnd_toEndOf="parent"app:layout_constraintStart_toStartOf="parent"app:layout_constraintTop_toBottomOf="@id/title" /><androidx.camera.view.PreviewViewandroid:id="@+id/previewView"android:layout_width="138dp"android:layout_height="104dp"android:layout_marginEnd="24dp"android:layout_marginBottom="24dp"android:background="@drawable/shape_corner_20dp_stroke_4dp_ffffff"android:foreground="@drawable/shape_corner_20dp_stroke_4dp_ffffff"android:visibility="gone"app:layout_constraintBottom_toBottomOf="parent"app:layout_constraintEnd_toEndOf="parent" /><com.agg.ui.AGGCircleImageViewandroid:id="@+id/flashPreview"android:layout_width="138dp"android:layout_height="104dp"android:layout_marginEnd="24dp"android:layout_marginBottom="24dp"android:contentDescription="@null"android:visibility="gone"app:borderColor="#FFFFC810"app:borderWidth="4dp"app:layout_constraintBottom_toBottomOf="parent"app:layout_constraintEnd_toEndOf="parent"app:radius="20dp" /><com.agg.ui.AGGIdentifyandroid:id="@+id/loading"android:layout_width="wrap_content"android:layout_height="wrap_content"app:UIContent="Open Camera..."app:layout_constraintBottom_toBottomOf="parent"app:layout_constraintEnd_toEndOf="parent"app:layout_constraintStart_toStartOf="parent"app:layout_constraintTop_toTopOf="parent" /></androidx.constraintlayout.widget.ConstraintLayout>

2.3 创建 AlgorithmLibHelper.kt

        访问条形码算法的帮助类，进行封装隔离，通过传入拍照后的图片路径，调用算法so库JNI模块。

object AlgorithmLibHelper {val productBeanLiveData = MutableLiveData<ProductBean>()private val TAG = AlgorithmLibHelper::class.java.simpleNameprivate var algorithmLib: AlgorithmLib? = nullfun init(context: Context) {Log.i(TAG, "init: ")val jsonFilePath = getJsonFilePath(context, "barcode_information_final.json")Log.i(TAG, "init: jsonFilePath=$jsonFilePath")algorithmLib = AlgorithmLib()algorithmLib?.initMatch(jsonFilePath)}fun identifyBarcode(imagePath: String) = runBlocking {Log.i(TAG, "identifyBarcode: imagePath=$imagePath")val identifyBarContent = algorithmLib?.matchBarcode(imagePath) ?: ""Log.i(TAG, "identifyBarcode: identifyBarContent=$identifyBarContent")if (identifyBarContent.isNotEmpty()) {try {val productInfo = GsonUtils.fromJson(identifyBarContent, ProductInfo::class.java)productBeanLiveData.postValue(productInfo.product_info)} catch (e: Exception) {e.printStackTrace()productBeanLiveData.postValue(ProductBean())}} else {productBeanLiveData.postValue(ProductBean())}}fun release() {Log.i(TAG, "release: ")algorithmLib = null}}

2.4 创建 AssetsHelper.kt

        由于算法库的商品数据库，采用的本地json数据库，所以当前技术方案就是通过app预先在源代码路径中放好json文件，然后动态将json文件拷贝到应用路径下，方便算法库读取与查询。

object AssetsHelper {fun getJsonFilePath(context: Context, assetName: String): String {val targetPath =AGGFileUtils.getMediaOutputDirectory(context as ContextWrapper).absolutePath + File.separator + assetNamecopyAssetToFile(context, assetName, targetPath)return targetPath}private fun copyAssetToFile(context: Context, assetName: String, targetPath: String) {val assetManager = context.assetsval inputStream: InputStream = assetManager.open(assetName)val file = File(targetPath)val outputStream = FileOutputStream(file)val buffer = ByteArray(1024)var read: Intwhile (inputStream.read(buffer).also { read = it } != -1) {outputStream.write(buffer, 0, read)}outputStream.flush()outputStream.close()inputStream.close()}}

        在src/main/assets/ 路径下放置 barcode_information_final.json 文件。

2.5 创建 ProductBean.kt

        商品信息的数据Bean，通过算法库返回的商品Json数据格式转换。

data class ProductBean(var product_name: String = "",var brand: String = "",var specifications: String = "",var price: String = "",var country_of_origin: String = "",var tax_rate: String = "",var tax_code: String = "",var function_and_use: String = "",
)data class ProductInfo(var product_info: ProductBean)/*
{"product_info": {"brand": "舒肤佳","country_of_origin": "中国","function_and_use": "用于日常清洁皮肤，有效去除污垢和细菌","price": "￥3.50","product_name": "香皂","specifications": "115G","tax_code": "1","tax_rate": "13%"}
}
*/

3. ⚛️ 算法so库JNI模块

1. 新建CMakeLists.txt，引入算法so库以及头文件；

2. 新建cpp文件，调用算法so库方法；

3. 新建native方法，加载JNI模块生成的库以及映射对应方法。

3.1 新建CMakeLists.txt，引入算法so库以及头文件

1）CMakeLists.txt 内容如下：

# 1. 设置 CMake 的最低版本要求
cmake_minimum_required(VERSION 3.22.1)# 2. 设置项目名称和支持的语言
project(BarcodeIdentify C CXX)# 3.1 指定头文件目录
include_directories(${PROJECT_SOURCE_DIR}/include)# 3.2 添加一个共享库目标：创建一个共享库来使用.so 库
add_library( # Sets the name of the library.barcode_match_jni# Sets the library as a shared library.SHARED# Provides a relative path to your source file(s).BarcodeMatchJni.cpp)
add_library( # Sets the name of the library.lib_barcode_match_interface# Sets the library as a shared library.SHARED# Provides a relative path to your source file(s).IMPORTED)
add_library( # Sets the name of the library.lib_barcode_match# Sets the library as a shared library.SHARED# Provides a relative path to your source file(s).IMPORTED)
add_library( # Sets the name of the library.lib_ZXing# Sets the library as a shared library.SHARED# Provides a relative path to your source file(s).IMPORTED)# 4. 设置 .so 文件的路径
set_target_properties(lib_barcode_match_interfacePROPERTIES IMPORTED_LOCATION${PROJECT_SOURCE_DIR}/lib/armeabi-v7a/libbarcode_match_interface.so)
set_target_properties(lib_barcode_matchPROPERTIES IMPORTED_LOCATION${PROJECT_SOURCE_DIR}/lib/armeabi-v7a/libbarcode_match.so)
set_target_properties(lib_ZXingPROPERTIES IMPORTED_LOCATION${PROJECT_SOURCE_DIR}/lib/armeabi-v7a/libZXing.so)# 5.1 链接系统库（例如 log 库）
find_library( # Sets the name of the path variable.log-lib# Specifies the name of the NDK library that# you want CMake to locate.log)
# 5.2 链接.so 库到目标
target_link_libraries( # Specifies the target library.barcode_match_jnilib_barcode_match_interfacelib_barcode_matchlib_ZXing# Links the target library to the log library# included in the NDK.${log-lib})

2）引入算法so库以及头文件

        算法 barcode_match_interface.h 头文件如下：

#ifndef BARCODE_MATCH_INTERFACE_H
#define BARCODE_MATCH_INTERFACE_H
#include <string>
class barcode_match_interface
{
private:void *barcode_match;
public:barcode_match_interface(/* args */);~barcode_match_interface();int init_barcode_match_interface(std::string json_input_path);std::string barcode_match_process(std::string input_img_path);
};#endif

3.2 新建cpp文件，调用算法so库方法

#include <jni.h>
#include <string>
#include <android/log.h>
#include "barcode_match_interface.h" // 包含算法库的头文件#define ALOGD(tag, ...) __android_log_print(ANDROID_LOG_DEBUG, tag, __VA_ARGS__)// 创建一个全局指针，用于存储 BarcodeMatch 类的实例
barcode_match_interface* barcodeMatchInstance = nullptr;// 实现 JNI 方法：initMatch
extern "C"
JNIEXPORT jint JNICALL
Java_com_agg_mocamera_portal_feature_demo_lib_AlgorithmLib_initMatch(JNIEnv* env, jobject thiz, jstring jsonInputPath) {const char* nativeJsonInputPath = env->GetStringUTFChars(jsonInputPath, nullptr);barcodeMatchInstance = new barcode_match_interface();int result = barcodeMatchInstance->init_barcode_match_interface(std::string(nativeJsonInputPath));env->ReleaseStringUTFChars(jsonInputPath, nativeJsonInputPath);if (result != 0) {delete barcodeMatchInstance;barcodeMatchInstance = nullptr;}return result;return 0;
}// 实现 JNI 方法：matchBarcode
extern "C"
JNIEXPORT jstring JNICALL
Java_com_agg_mocamera_portal_feature_demo_lib_AlgorithmLib_matchBarcode(JNIEnv *env, jobject thiz, jstring inputImagePath) {const char* nativeInputImagePath = env->GetStringUTFChars(inputImagePath, nullptr);std::string result;if (barcodeMatchInstance != nullptr) {result = barcodeMatchInstance->barcode_match_process(std::string(nativeInputImagePath));} else {result = "Error: BarcodeMatch instance not initialized";}ALOGD("TAG-AGG","%s", result.c_str());env->ReleaseStringUTFChars(inputImagePath, nativeInputImagePath);return env->NewStringUTF(result.c_str());
}

3.3 新建native方法，加载JNI模块生成的库以及映射对应方法

class AlgorithmLib {external fun initMatch(jsonInputPath: String): Intexternal fun matchBarcode(imagePath: String): Stringcompanion object {init {System.loadLibrary("barcode_match_jni")}}}

3.4 配置 build.gradle 文件

        在build.gradle文件中，确保项目支持所需的ABI架构。

android {defaultConfig {ndk {abiFilters 'armeabi-v7a'}}

        在 build.gradle 文件中配置 CMake。

android {externalNativeBuild {cmake {path "src/main/cpp/CMakeLists.txt"}}

注：对于算法条形码识别模块，由于篇幅与技术栈问题，本文就不再赘述。

4. ✅ 小结

        对于条形码识别，本文只是一个基础实现方案，更多业务细节请参考产品逻辑去实现。

        另外，由于本人能力有限，如有错误，敬请批评指正，谢谢。

---