FeverCam

The PapillonFeverCam library contains the tools required to build a FeverCam solution for person temperature remote measurement. DigitalBarriers FeverCam is made of 2 camera heads (one thermal and one color) which share the same field of view (approximately) so that it is possible to match persons in the color image with temperature data retrieved from the thermal camera.

The main classes to use are:

• Input Video Stream Specialised PInputVideoStream for reading temperature video streams and controlling thermal camera parameters (black body location, reference temperatures...)
• PFeverCamCalibration: Methods to associate images produced by DigitalBarriers FeverCam color and thermal camera heads. This ensures that user can read temperature at any point in the color images.
• PFeverCamProcessor: Main class for handling FeverCam processing for person temperature reading. It requires a valid PFeverCamCalibration to perform correctly. This class handles:
  • Temperature image acquisition and synchronization with color images
  • Human temperature estimation (measured skin temperature is corrected) from faces detected in color images
  • Black-body localization to help configuration or correct potential camera movements

PFeverCamProcessor is based on the following classes which can be used individually as well:

• PFeverCamAnalytics: Analytics functions to process FeverCam images (change detection and black body localization).
• PFeverCamTemperatureCorrection: Human skin temperature correction function for feverCam readings.
• PFeverCamTemperatureImagesHandler: FeverCam temperature images capture, processing and handling.

Example - Calibration

/*
 * Copyright (C) 2015-2018 Digital Barriers plc. All rights reserved.
 * Contact: http://www.digitalbarriers.com/
 *
 * This file is part of the Papillon SDK.
 *
 * You can't use, modify or distribute any part of this file without
 * the explicit written agreements of Digital Barriers.
 */

#include <PapillonCore.h>
#include <PImageInternal.h>
#include <PFeverCamCalibration.h>
#include <opencv2/core.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/highgui/highgui.hpp>

USING_NAMESPACE_PAPILLON


const PString SAMPLE_DIR = PPath::Join(PUtils::GetEnv("PAPILLON_INSTALL_DIR"), "Data", "Samples");

// Global variables
PFeverCamCalibration fccalib;
PPoint2DfList        colorPts, thermPts;
cv::Mat              colorImg, thermImg;
cv::Mat              colorImgSrc, thermImgSrc;
PImage               refColor, refTherm;


// Simple cross drawing function, cross size and width adapts to image height
void drawCross(cv::Mat& io_img, const PPoint2Df& i_pt, const cv::Scalar& i_col)
{
    int s = 1 + io_img.rows / 100;
    cv::line(io_img, cv::Point(i_pt.GetX() - s, i_pt.GetY() - s), cv::Point(i_pt.GetX() + s, i_pt.GetY() + s), i_col, 1 + s/5);
    cv::line(io_img, cv::Point(i_pt.GetX() + s, i_pt.GetY() - s), cv::Point(i_pt.GetX() - s, i_pt.GetY() + s), i_col, 1 + s/5);
}


// Updates display (show crosses where user has clicked)
// computes calibration (and saves it) and displays fused image (if possible)
void updateCalibAndDisplay()
{
    colorImgSrc.copyTo(colorImg);
    thermImgSrc.copyTo(thermImg);
    for(int i = 0; i < colorPts.Size(); ++i)
        drawCross(colorImg, colorPts.Get(i), cv::Scalar(0, 0, 255));
    for(int i = 0; i < thermPts.Size(); ++i)
        drawCross(thermImg, thermPts.Get(i), cv::Scalar(0, 0, 255));
    cv::imshow("color", colorImg);
    cv::imshow("therm", thermImg);

    PResult r = fccalib.UpdateCalibration(colorPts, thermPts);
    if (r.Ok())
    {
        fccalib.SaveToFile(PPath::Join(SAMPLE_DIR, "calibrationMatrix.bin")).LogErrorIfAny("saving calibration");
        PImage fused;
        fccalib.FuseImages(refColor, refTherm, fused);
        fused.DisplayScaled("Fused", 1280, 768);
    }
}


// left mouse click callback => calls updateCalibDisplay
void ptSelectionCallBack(int e, int x, int y, int flags, void* udata)
{
    if (e == cv::EVENT_LBUTTONUP)
    {
        PPoint2DfList* l = (PPoint2DfList*)(udata);
        l->Add(PPoint2Df(x, y));
        updateCalibAndDisplay();
    }
}


int main()
{
    PapillonSDK::Initialise().OrDie();

    // Load sample images
    PString colorPath = PPath::Join(SAMPLE_DIR, "feverCam_color.png"),
            thermPath = PPath::Join(SAMPLE_DIR, "feverCam_therm.png");

    refColor.Load(colorPath).LogErrorIfAny("loading color");
    refTherm.Load(thermPath).LogErrorIfAny("loading therm");
    colorImgSrc = cv::imread(colorPath.c_str());
    thermImgSrc = cv::imread(thermPath.c_str());

    // Create "color" and "therm" windows and adapt size/callbacks etc as needed
    cv::namedWindow("color", cv::WINDOW_NORMAL | cv::WINDOW_KEEPRATIO);
    cv::namedWindow("therm", cv::WINDOW_NORMAL | cv::WINDOW_KEEPRATIO);

    cv::setMouseCallback("color", ptSelectionCallBack, &colorPts);
    cv::setMouseCallback("therm", ptSelectionCallBack, &thermPts);

    cv::resizeWindow("color", 1280, 768);
    cv::resizeWindow("therm", 1024, 768);

    // Show images
    cv::imshow("color", colorImgSrc);
    cv::imshow("therm", thermImgSrc);

    // wait for any key press in one of the color/therm images
    cv::waitKey(-1);

    PapillonSDK::Finalise();
    return 0;
}

Example - Processing

/*
 * Copyright (C) 2015-2018 Digital Barriers plc. All rights reserved.
 * Contact: http://www.digitalbarriers.com/
 *
 * This file is part of the Papillon SDK.
 *
 * You can't use, modify or distribute any part of this file without
 * the explicit written agreements of Digital Barriers.
 */

#include <PFeverCamCalibration.h>
#include <PFeverCamProcessor.h>
#include <PapillonCore.h>
USING_NAMESPACE_PAPILLON

const PString SAMPLE_DIR = PPath::Join(PUtils::GetEnv("PAPILLON_INSTALL_DIR"), "Data", "Samples");

int main(int argc, char** argv) {
    PapillonSDK::Initialise().OrDie();

    // Handle standard options
    POption opt(argc, argv);
    opt.AddStandardOptions();  // set-up logging
    PString thermIP  = opt.String("therm,t", "192.168.1.101", "FeverCam thermal camera IP address");
    PString colorIP  = opt.String("color,c", "192.168.1.100", "FeverCam color camera IP address");
    PString colorPwd = opt.String("password,p", "admin", "FeverCam color camera \"admin\" password");

    PFeverCamCalibration fccalib;
    PFeverCamProcessor   fcproc;

    // Configure a default FaceLog6 analytics that will be used for detection
    PAnalytics  faceLog;
    PProperties faceLogParameters;
    faceLogParameters.Set("faceDetector.MinDetectionSize", 80);
    faceLogParameters.Set("faceDetector.MaxDetectionSize", 1000);
    faceLogParameters.Set("faceDetector.Threshold", 0.0);
    faceLogParameters.Set("faceDetector.regionOfInterest", papillon::PRectanglei(0, 0, 0, 0));
    faceLogParameters.Set("faceDetector.enableLocaliser", true);
    faceLogParameters.Set("faceDetector.MinCertainty", 0.0);
    faceLogParameters.Set("faceDetector.MaxLength", 100);
    faceLogParameters.Set("faceDetector.MinLength", 1);
    faceLogParameters.Set("faceDetector.MaxGap", 10);
    faceLogParameters.Set("faceDetector.SightingSimilarity", 0.6);
    faceLogParameters.Set("faceDetector.FrameLag", 50);
    faceLogParameters.Set("faceDetector.SpatialFilter", true);
    faceLogParameters.Set("faceDetector.Motion", false);
    faceLogParameters.Set("faceDetector.OSDFlags", 1);
    PWatchlistOptions watchlistOptions;
    watchlistOptions.SetTopN(1);
    watchlistOptions.SetThreshold(0.75);
    watchlistOptions.SetNormalise(false);
    faceLogParameters.Set("faceDetector.WatchlistOptions", watchlistOptions);
    faceLogParameters.Set("faceDetector.MaxFaceDetectorFR", -1);
    faceLogParameters.Set("gpuId", -2);  // Use optimized CPU models
    PAnalytics::Create("FaceLog6", faceLogParameters, faceLog).OrDie("Creating simple face detector");

    // Load calibration for the processor
    fccalib.LoadFromFile(PPath::Join(SAMPLE_DIR, "calibrationMatrix.bin")).OrDie("Loading calibration matrix");
    fcproc.SetCalibration(fccalib);
    // Connect to feverCam
    fcproc.HandleFeverCam(thermIP).OrDie("Connecting to feverCam");
    // Connect to color camera
    PInputVideoStream colorIvs;
    PInputVideoStream::Open(PString("rtsp://admin:%1@%2/stream0").Arg(colorPwd).Arg(colorIP), colorIvs)
        .OrDie("Opening color stream");

    PImage img;

    PFrame colorFame, thermFrame;
    // Detection will run as long as video streams are available
    while(colorIvs.GetFrame(colorFame, 30000).LogErrorIfAny("retrieving color frame").Ok() && fcproc.IsActive()) {
        PList faceEvents;
        faceLog.Apply(colorFame, faceEvents).OrDie("processing color frame");
        // Here we can detect black body should it be needed
        PList bbodyCandidates;
        if(fcproc.LocateBlackBody(colorFame, bbodyCandidates).LogErrorIfAny("Locating black body").Ok() &&
           !bbodyCandidates.IsEmpty()) {
            P_LOG_INFO << "Black body candidates: " << bbodyCandidates.ToString();
            // At this point user could update automatically black body position in the camera
            // To do this (very simple example assuming only one candidate is found):
            //
            // Retrieve the thermal video stream
            // PInputVideoStream tivs;
            // fcproc.GetTemperatureVideoSource(tivs).OrDie("Retrieving thermal video source");
            // PRectanglef bbodyRect;
            // bbodyCandidates.Get(0, bbodyRect).OrDie("Retrieving first black body candidate");
            // tivs.Set("BLACKBODY_RECTANGLE", bbodyRect).OrDie("Setting black body position");
        }
        // Display synchronized temperature image
        if(fcproc.GetThermalFrameAt(colorFame.GetTimestampUTC(), thermFrame).Ok())
            thermFrame.Display("thermal", 0);

        // Retrieve image that will be used for display of faces and temperature
        img = colorFame.GetImageShared(PImage::E_RGB8U).Clone();
        // Now process faces that we may have detected
        for(int32 e = 0; e < faceEvents.Size(); ++e) {
            PEvent evt;
            faceEvents.Get(e, evt).OrDie("Retrieving event");
            // We only process faces
            if(evt.GetType() == "Face") {
                // Extract the detection
                const PProperties& eventProperties = evt.GetPayload();
                PDetection         detection;
                if(eventProperties.Get("Detection", detection).LogIfError().Failed())
                    continue;

                // We'll draw the image with the detection on it
                PUtils::DrawLabel(img, detection, "", PUtils::E_TOP_CENTRE, PUtils::E_BLACK, 1., 1, PColour3i::Red());

                // Estimate face temperature for this face
                double temperature, confidence;
                if(fcproc.EstimateFaceTemperature(detection, temperature, confidence)
                       .LogErrorIfAny("Estimating face temperature")) {
                    // We managed to get face temperature (so calibration was correct enough that we could match face to
                    // thermal equivalent)
                    P_LOG_INFO << "Face detected at " << detection.GetFrame().GetTimestampUTC().ToStringISOWithMs()
                               << " has temperature " << temperature << "C with confidence " << confidence;

                    // we'll add the color information on top
                    PUtils::DrawLabel(img, detection, PString("T: %1C (conf: %2)").Arg(temperature).Arg(confidence),
                                      PColour3i::Darkgray(), PColour3i::Red(), papillon::PUtils::E_TOP_CENTRE, 1., 0,
                                      papillon::PColour3i::White(), papillon::PImage::E_FONT_HERSHEY_TRIPLEX);
                }
            }
        }
        // Display image with detection and temperature if available
        img.Display("Detection", 0);
    }

    PapillonSDK::Finalise();
    return 0;
}

Using the Library

• Add PapillonFeverCam to your linker.
• Add ${PAPILLON_INSTALL_DIR}/include/PapillonFeverCam to your include path.