We think VIM3 C++ Demo is too complex. It is not friendly for users. So we provide a lite version. This document will help you use this lite version.

Densenet_CTC is a text recognition model. It only can recognize single line text. Therefore usually, it needs to be used in conjunction with a text detection model.

Recognition image and inference results on VIM3.

We will use a DenseNet model based on YCG09/chinese_ocr.

git clone https://github.com/YCG09/chinese_ocr

We provided a docker image which contains the required environment to convert the model.

Follow Docker official docs to install Docker: Install Docker Engine on Ubuntu.

Follow the command below to get Docker image:

docker pull numbqq/npu-vim3

$ git lfs install
$ git lfs clone https://github.com/khadas/aml_npu_sdk.git

$ cd aml_npu_sdk/acuity-toolkit/demo && ls
aml_npu_sdk/acuity-toolkit/demo$ ls
0_import_model.sh  1_quantize_model.sh  2_export_case_code.sh  data  dataset_npy.txt  dataset.txt  extractoutput.py  inference.sh  input.npy  model

export.py

import onnx
from keras.models import *
import keras
import keras2onnx
from train import get_model
import densenet
 
basemodel, model = get_model(32, 88) # input height, classes number
basemodel.load_weights("models/weights_densenet-32-0.40.h5")
onnx_model = keras2onnx.convert_keras(basemodel, basemodel.name, target_opset=12)
onnx_model.graph.input[0].type.tensor_type.shape.dim[0].dim_value = int(1)
onnx_model.graph.input[0].type.tensor_type.shape.dim[1].dim_value = int(1)
onnx_model.graph.input[0].type.tensor_type.shape.dim[2].dim_value = int(32)
onnx_model.graph.input[0].type.tensor_type.shape.dim[3].dim_value = int(280)
onnx_model.graph.output[0].type.tensor_type.shape.dim[0].dim_value = int(1)
onnx_model.graph.node.remove(onnx_model.graph.node[0])
onnx_model.graph.node[0].input[0] = "the_input"
onnx.save_model(onnx_model, "./densenet_ctc.onnx")

Enter aml_npu_sdk/acuity-toolkit/demo and put densenet_ctc.onnx into demo/model. Modify 0_import_model.sh as follows.

0_import_model.sh

#!/bin/bash
 
NAME=densenet_ctc
ACUITY_PATH=../bin/
 
pegasus=${ACUITY_PATH}pegasus
if [ ! -e "$pegasus" ]; then
    pegasus=${ACUITY_PATH}pegasus.py
fi
 
#Onnx
$pegasus import onnx \
    --model  ./model/${NAME}.onnx \
    --output-model ${NAME}.json \
    --output-data ${NAME}.data 
 
#generate inpumeta  --source-file dataset.txt
$pegasus generate inputmeta \
	--model ${NAME}.json \
	--input-meta-output ${NAME}_inputmeta.yml \
	--channel-mean-value "0 0 0 0.0039215"  \
	--source-file dataset.txt

Run 0_import_model.sh.

$ bash 0_import_model.sh

After running, densenet_ctc.json will be generated. Modify it as follows.

densenet_ctc.json

        "Flatten_flatten_2_7": {
            "name": "Flatten_flatten_2",
            "op": "reshape",
            "parameters": {
                "shape": [
-                   0,
+                   1,
                    -1
                ]
            },
            "inputs": [
                "@Reshape_flatten_reshape_0_8:out0"
            ],
            "outputs": [
                "out0"
            ]
        },

Then, modify 1_quantize_model.sh and 2_export_case_code.sh and run.

1_quantize_model.sh

#!/bin/bash
 
NAME=densenet_ctc
ACUITY_PATH=../bin/
 
pegasus=${ACUITY_PATH}pegasus
if [ ! -e "$pegasus" ]; then
    pegasus=${ACUITY_PATH}pegasus.py
fi
 
#--quantizer asymmetric_affine --qtype  uint8
#--quantizer dynamic_fixed_point  --qtype int8(int16,note s905d3 not support int16 quantize) 
# --quantizer perchannel_symmetric_affine --qtype int8(int16, note only T3(0xBE) can support perchannel quantize)
$pegasus  quantize \
	--quantizer dynamic_fixed_point \
	--qtype int8 \
	--rebuild \
	--with-input-meta  ${NAME}_inputmeta.yml \
	--model  ${NAME}.json \
	--model-data  ${NAME}.data

2_export_case_code.sh

#!/bin/bash
 
NAME=densenet_ctc
ACUITY_PATH=../bin/
 
pegasus=$ACUITY_PATH/pegasus
if [ ! -e "$pegasus" ]; then
    pegasus=$ACUITY_PATH/pegasus.py
fi
 
$pegasus export ovxlib\
    --model ${NAME}.json \
    --model-data ${NAME}.data \
    --model-quantize ${NAME}.quantize \
    --with-input-meta ${NAME}_inputmeta.yml \
    --dtype quantized \
    --optimize VIPNANOQI_PID0X88  \
    --viv-sdk ${ACUITY_PATH}vcmdtools \
    --pack-nbg-unify
 
rm -rf ${NAME}_nbg_unify
 
mv ../*_nbg_unify ${NAME}_nbg_unify
 
cd ${NAME}_nbg_unify
 
mv network_binary.nb ${NAME}.nb
 
cd ..
 
#save normal case demo export.data 
mkdir -p ${NAME}_normal_case_demo
mv  *.h *.c .project .cproject *.vcxproj BUILD *.linux *.export.data ${NAME}_normal_case_demo
 
# delete normal_case demo source
#rm  *.h *.c .project .cproject *.vcxproj  BUILD *.linux *.export.data
 
rm *.data *.quantize *.json *_inputmeta.yml

If you use VIM3L, optimize use VIPNANOQI_PID0X99.

If run succeed, converted model and library will generate in demo/densenet_ctc_nbg_unify.

$ bash 0_import_model.sh && bash 1_quantize_model.sh && bash 2_export_case_code.sh
$ cd acuity-toolkit/demo/densenet_ctc_nbg_unify
$ ls
BUILD  densenet_ctc.nb  densenetctc.vcxproj  main.c  makefile.linux  nbg_meta.json  vnn_densenetctc.c  vnn_densenetctc.h  vnn_global.h  vnn_post_process.c  vnn_post_process.h  vnn_pre_process.c  vnn_pre_process.h

Get the source code: khadas/vim3_npu_applications_lite

$ git clone https://github.com/khadas/vim3_npu_applications_lite

$ sudo apt update
$ sudo apt install libopencv-dev python3-opencv cmake

Put densenet_ctc.nb into vim3_npu_applications_lite/densenet_ctc_demo_picture/nn_data. Replace densenet_ctc_demo_picture/vnn_densenetctc.c and densenet_ctc_demo_picture/include/vnn_densenetctc.h with your generating vnn_densenetctc.c and vnn_densenetctc.h.

# Compile
$ cd vim3_npu_applications_lite/densenet_ctc_demo_picture
$ bash build_vx.sh
$ cd bin_r_cv4
$ ./densenet_ctc_demo_picture -m ../nn_data/densenet_ctc.nb -p ../KhadasTeam.png