android - How to process pose, left hand, right hand, and face landmarks in the correct order using MediaPipe? - Stack Overflow

admin管理员组

文章数量:1124559

I am working on a project where I need to process pose landmarks, left hand landmarks, right hand landmarks, and face landmarks in a specific order using MediaPipe. However, the landmarks are not being processed in the correct sequence. Instead of following the order "pose -> left hand -> right hand -> face," the camera seems to be capturing and processing the landmarks in a different order, often starting with the right hand landmarks.

Here is a brief overview of my setup:

I am using MediaPipe to capture the landmark data. I have implemented callbacks for each type of landmark (pose, left hand, right hand, face). I need the landmark data to be processed in the order: pose -> left hand -> right hand -> face. The callback functions are being triggered, but the order of processing is incorrect. I would appreciate any guidance or examples on how to ensure the landmarks are processed in the correct sequence. Specifically:

How can I enforce the correct order of processing the landmarks? Is there a way to buffer or temporarily store the landmarks until they can be processed in the desired order? Here is a snippet of my current implementation for reference:

 private void startLandmarkPacketCallbacks() {
        // Pose landmarks callback
        processor.addPacketCallback(
            "pose_landmarks",
            (packet) -> {
                synchronized (landmarkLock) {
                    if (packet != null) {
                        try {
                            NormalizedLandmarkList landmarks = PacketGetter.getProto(packet, NormalizedLandmarkList.class);
                            if (landmarks != null) {
                                currentPoseLandmarks = landmarks;
                                checkFrameComplete();
                            }
                        } catch (InvalidProtocolBufferException e) {
                            Log.e(TAG, "Pose landmarks error: " + e.getMessage());
                        }
                    }
                }
            }
        );

    // Left hand landmarks callback
    processor.addPacketCallback(
        "left_hand_landmarks",
        (packet) -> {
            synchronized (landmarkLock) {
                if (packet != null) {
                    try {
                        NormalizedLandmarkList landmarks = PacketGetter.getProto(packet, NormalizedLandmarkList.class);
                        if (landmarks != null) {
                            currentLeftHandLandmarks = landmarks;
                            checkFrameComplete();
                        }
                    } catch (InvalidProtocolBufferException e) {
                        Log.e(TAG, "Left hand landmarks error: " + e.getMessage());
                    }
                }
            }
        }
    );

    // Right hand landmarks callback
    processor.addPacketCallback(
        "right_hand_landmarks",
        (packet) -> {
            synchronized (landmarkLock) {
                if (packet != null) {
                    try {
                        NormalizedLandmarkList landmarks = PacketGetter.getProto(packet, NormalizedLandmarkList.class);
                        if (landmarks != null) {
                            currentRightHandLandmarks = landmarks;
                            checkFrameComplete();
                        }
                    } catch (InvalidProtocolBufferException e) {
                        Log.e(TAG, "Right hand landmarks error: " + e.getMessage());
                    }
                }
            }
        }
    );

    // Face landmarks callback
    processor.addPacketCallback(
        "face_landmarks",
        (packet) -> {
            synchronized (landmarkLock) {
                if (packet != null) {
                    try {
                        NormalizedLandmarkList landmarks = PacketGetter.getProto(packet, NormalizedLandmarkList.class);
                        if (landmarks != null) {
                            currentFaceLandmarks = landmarks;
                            checkFrameComplete();
                        }
                    } catch (InvalidProtocolBufferException e) {
                        Log.e(TAG, "Face landmarks error: " + e.getMessage());
                    }
                }
            }
        }
    );
}

private void checkFrameComplete() {
    if (currentPoseLandmarks != null && currentLeftHandLandmarks != null &&
        currentRightHandLandmarks != null && currentFaceLandmarks != null) {

        // Add to queues
        poseLandmarksQueue.offer(currentPoseLandmarks);
        leftHandLandmarksQueue.offer(currentLeftHandLandmarks);
        rightHandLandmarksQueue.offer(currentRightHandLandmarks);
        faceLandmarksQueue.offer(currentFaceLandmarks);

        // Limit queue sizes
        while (poseLandmarksQueue.size() > LANDMARK_HISTORY_SIZE) poseLandmarksQueue.poll();
        while (leftHandLandmarksQueue.size() > LANDMARK_HISTORY_SIZE) leftHandLandmarksQueue.poll();
        while (rightHandLandmarksQueue.size() > LANDMARK_HISTORY_SIZE) rightHandLandmarksQueue.poll();
        while (faceLandmarksQueue.size() > LANDMARK_HISTORY_SIZE) faceLandmarksQueue.poll();

        // Check if there is enough data
        if (poseLandmarksQueue.size() == LANDMARK_HISTORY_SIZE &&
            leftHandLandmarksQueue.size() == LANDMARK_HISTORY_SIZE &&
            rightHandLandmarksQueue.size() == LANDMARK_HISTORY_SIZE &&
            faceLandmarksQueue.size() == LANDMARK_HISTORY_SIZE) {

            processLandmarks();
        }

        resetCurrentFrame();
    }
}

private void resetCurrentFrame() {
    currentPoseLandmarks = null;
    currentLeftHandLandmarks = null;
    currentRightHandLandmarks = null;
    currentFaceLandmarks = null;
}

Thanks for help!

本文标签：