1. React Native
  2. On-Device AI Scanning (OCR)

React Native

On-Device AI Scanning (OCR)

The Vision SDK allows for on-device AI scanning (OCR), enabling offline extraction of structured information from documents such as shipping labels. This is ideal when low latency and offline functionality are critical, such as in warehouse and logistics environments.

🛠️ Step 1: Preparing On-Device OCR

Before using the on-device OCR, you must prepare the model using the imperative method:configureOnDeviceModel and set ocrMode prop to 'onDevice'. This ensures the necessary AI models are downloaded and ready to use.

Model Size and Type Configuration

If you want Vision SDK to automatically determine the best model size based on your PackageX subscription:

        import VisionSdkView, { VisionSdkRefProps } from 'react-native-vision-sdk'
import { useRef, useEffect } from 'react'

const Example = () => {
    const vsdkRef = useRef<VisionSdkRefProps>(null);
    useEffect(() => {
        const timeout = setTimeout(() => {
            vsdkRef.current.startRunningHandler() // this method starts the camera feed
            vsdkRef.current.configureOnDeviceModel({
                type: 'shipping_label', // can be one of ['shipping_label', 'item_label', 'bill_of_lading', 'document_classification']
                size: 'large' //this is an optional parameter, exclude if you want the sdk to decide the appropriate size by itself. Can be one of ['large', 'micro']
            })
            
        }, 0)

        return () => {
            clearTimeout(timeout)
        }
    }, [])



    return (
        <View style={{flex: 1}}>
            <VisionSdkView 
                ref={vsdkRef}
                mode='ocr'
                ocrMode='on_device'
                ocrType='shipping_label'

            />
        </View>
    )
}
NOTE

Please note that you need to wait before the model is successfully downloaded and configured. This would be a one time operation only. More on this in the next section.

🧩 Model download progress events

You need to set onModelDownloadProgress event handler to receive download progress events

        const Example = () => {
    const vsdkRef = useRef<VisionSdkRefProps>(null);
    useEffect(() => {
        const timeout = setTimeout(() => {
            vsdkRef.current.startRunningHandler() // this method starts the camera feed
            vsdkRef.current.configureOnDeviceModel({
                type: 'shipping_label', // can be one of ['shipping_label', 'item_label', 'bill_of_lading', 'document_classification']
                size: 'large' //this is an optional parameter, exclude if you want the sdk to decide the appropriate size by itself. Can be one of ['large', 'micro']
            })
            
        }, 0)

        return () => {
            clearTimeout(timeout)
        }
    }, [])

    const handleDownloadProgress = (data) => {
        /*
        The data argument has the following structure:
        {
            progress: number,  //download percentage
            downloadStatus: boolean, //whether the model is downloaded,
            isReady: boolean // whether the model is ready for use
        }
        */

    }


    return (
        <View style={{flex: 1}}>
            <VisionSdkView 
                ref={vsdkRef}
                mode='ocr'
                ocrMode='on_device'
                ocrType='shipping_label'
                onModelDownloadProgress={handleDownloadProgress}

            />
        </View>
    )
}

📝 Make sure the model is prepared successfully before scanning. Each model may vary in size and complexity depending on the document type.

🧠 Step 2: Extracting Data from Image

Once the model is prepared successfully, make sure you have ocrMode and ocrType props set appropriately. After that you can call imperative method cameraCaptureHandler to capture the image, the response will be available in onOCRScan event handler.

        const Example = () => {
    const vsdkRef = useRef<VisionSdkRefProps>(null);
    useEffect(() => {
        const timeout = setTimeout(() => {
            vsdkRef.current.startRunningHandler() // this method starts the camera feed
            vsdkRef.current.configureOnDeviceModel({
                type: 'shipping_label', // can be one of ['shipping_label', 'item_label', 'bill_of_lading', 'document_classification']
                size: 'large' //this is an optional parameter, exclude if you want the sdk to decide the appropriate size by itself. Can be one of ['large', 'micro']
            })
            
        }, 0)

        return () => {
            clearTimeout(timeout)
        }
    }, [])

    const handleDownloadProgress = (data) => {
        /*
        The data argument has the following structure:
        {
            progress: number,  //download percentage
            downloadStatus: boolean, //whether the model is downloaded,
            isReady: boolean // whether the model is ready for use
        }
        */

    }

    const handleCapture = () => {
        vsdkRef?.current?.cameraCaptureHandler()
    }

    const handleOCRScan = (event) => {
        //event.data contains the extracted information from the document
    }

    return (
        <View style={{flex: 1}}>
            <VisionSdkView 
                ref={vsdkRef}
                mode='ocr'
                ocrMode='on_device'
                ocrType='shipping_label'
                onModelDownloadProgress={handleDownloadProgress}
                onOCRScan={handleOCRScan}
            />

            <TouchableOpacity onPress={handleCapture}>
                <Text>Capture</Text>
            </TouchableOpacity>
        </View>
    )
}

🔁 The returned data follows the same structure as the PackageX Cloud OCR API response.

Previous <- AI Scanning APIs
Next Error Reporting ->