FeverCam

The PapillonFeverCam library contains the tools required to build a FeverCam solution for person temperature remote measurement. DigitalBarriers FeverCam is made of 2 camera heads (one thermal and one color) which share the same field of view (approximately) so that it is possible to match persons in the color image with temperature data retrieved from the thermal camera.

The main classes to use are:

  • Input Video Stream Specialised PInputVideoStream for reading temperature video streams and controlling thermal camera parameters (black body location, reference temperatures...)
  • PFeverCamCalibration: Methods to associate images produced by DigitalBarriers FeverCam color and thermal camera heads. This ensures that user can read temperature at any point in the color images.
  • PFeverCamProcessor: Main class for handling FeverCam processing for person temperature reading. It requires a valid PFeverCamCalibration to perform correctly. This class handles:
    • Temperature image acquisition and synchronization with color images
    • Human temperature estimation (measured skin temperature is corrected) from faces detected in color images
    • Black-body localization to help configuration or correct potential camera movements

PFeverCamProcessor is based on the following classes which can be used individually as well:

Example - Calibration

/*
* Copyright (C) 2015-2018 Digital Barriers plc. All rights reserved.
* Contact: http://www.digitalbarriers.com/
*
* This file is part of the Papillon SDK.
*
* You can't use, modify or distribute any part of this file without
* the explicit written agreements of Digital Barriers.
*/
#include <PapillonCore.h>
#include <PImageInternal.h>
#include <opencv2/core.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/highgui/highgui.hpp>
USING_NAMESPACE_PAPILLON
const PString SAMPLE_DIR = PPath::Join(PUtils::GetEnv("PAPILLON_INSTALL_DIR"), "Data", "Samples");
// Global variables
PPoint2DfList colorPts, thermPts;
cv::Mat colorImg, thermImg;
cv::Mat colorImgSrc, thermImgSrc;
PImage refColor, refTherm;
// Simple cross drawing function, cross size and width adapts to image height
void drawCross(cv::Mat& io_img, const PPoint2Df& i_pt, const cv::Scalar& i_col)
{
int s = 1 + io_img.rows / 100;
cv::line(io_img, cv::Point(i_pt.GetX() - s, i_pt.GetY() - s), cv::Point(i_pt.GetX() + s, i_pt.GetY() + s), i_col, 1 + s/5);
cv::line(io_img, cv::Point(i_pt.GetX() + s, i_pt.GetY() - s), cv::Point(i_pt.GetX() - s, i_pt.GetY() + s), i_col, 1 + s/5);
}
// Updates display (show crosses where user has clicked)
// computes calibration (and saves it) and displays fused image (if possible)
void updateCalibAndDisplay()
{
colorImgSrc.copyTo(colorImg);
thermImgSrc.copyTo(thermImg);
for(int i = 0; i < colorPts.Size(); ++i)
drawCross(colorImg, colorPts.Get(i), cv::Scalar(0, 0, 255));
for(int i = 0; i < thermPts.Size(); ++i)
drawCross(thermImg, thermPts.Get(i), cv::Scalar(0, 0, 255));
cv::imshow("color", colorImg);
cv::imshow("therm", thermImg);
PResult r = fccalib.UpdateCalibration(colorPts, thermPts);
if (r.Ok())
{
fccalib.SaveToFile(PPath::Join(SAMPLE_DIR, "calibrationMatrix.bin")).LogErrorIfAny("saving calibration");
PImage fused;
fccalib.FuseImages(refColor, refTherm, fused);
fused.DisplayScaled("Fused", 1280, 768);
}
}
// left mouse click callback => calls updateCalibDisplay
void ptSelectionCallBack(int e, int x, int y, int flags, void* udata)
{
if (e == cv::EVENT_LBUTTONUP)
{
PPoint2DfList* l = (PPoint2DfList*)(udata);
l->Add(PPoint2Df(x, y));
updateCalibAndDisplay();
}
}
int main()
{
// Load sample images
PString colorPath = PPath::Join(SAMPLE_DIR, "feverCam_color.png"),
thermPath = PPath::Join(SAMPLE_DIR, "feverCam_therm.png");
refColor.Load(colorPath).LogErrorIfAny("loading color");
refTherm.Load(thermPath).LogErrorIfAny("loading therm");
colorImgSrc = cv::imread(colorPath.c_str());
thermImgSrc = cv::imread(thermPath.c_str());
// Create "color" and "therm" windows and adapt size/callbacks etc as needed
cv::namedWindow("color", cv::WINDOW_NORMAL | cv::WINDOW_KEEPRATIO);
cv::namedWindow("therm", cv::WINDOW_NORMAL | cv::WINDOW_KEEPRATIO);
cv::setMouseCallback("color", ptSelectionCallBack, &colorPts);
cv::setMouseCallback("therm", ptSelectionCallBack, &thermPts);
cv::resizeWindow("color", 1280, 768);
cv::resizeWindow("therm", 1024, 768);
// Show images
cv::imshow("color", colorImgSrc);
cv::imshow("therm", thermImgSrc);
// wait for any key press in one of the color/therm images
cv::waitKey(-1);
return 0;
}

Example - Processing

/*
* Copyright (C) 2015-2018 Digital Barriers plc. All rights reserved.
* Contact: http://www.digitalbarriers.com/
*
* This file is part of the Papillon SDK.
*
* You can't use, modify or distribute any part of this file without
* the explicit written agreements of Digital Barriers.
*/
#include <PapillonCore.h>
USING_NAMESPACE_PAPILLON
const PString SAMPLE_DIR = PPath::Join(PUtils::GetEnv("PAPILLON_INSTALL_DIR"), "Data", "Samples");
int main(int argc, char** argv) {
// Handle standard options
POption opt(argc, argv);
opt.AddStandardOptions(); // set-up logging
PString thermIP = opt.String("therm,t", "192.168.1.101", "FeverCam thermal camera IP address");
PString colorIP = opt.String("color,c", "192.168.1.100", "FeverCam color camera IP address");
PString colorPwd = opt.String("password,p", "admin", "FeverCam color camera \"admin\" password");
// Configure a default FaceLog6 analytics that will be used for detection
PAnalytics faceLog;
PProperties faceLogParameters;
faceLogParameters.Set("faceDetector.MinDetectionSize", 80);
faceLogParameters.Set("faceDetector.MaxDetectionSize", 1000);
faceLogParameters.Set("faceDetector.Threshold", 0.0);
faceLogParameters.Set("faceDetector.regionOfInterest", papillon::PRectanglei(0, 0, 0, 0));
faceLogParameters.Set("faceDetector.enableLocaliser", true);
faceLogParameters.Set("faceDetector.MinCertainty", 0.0);
faceLogParameters.Set("faceDetector.MaxLength", 100);
faceLogParameters.Set("faceDetector.MinLength", 1);
faceLogParameters.Set("faceDetector.MaxGap", 10);
faceLogParameters.Set("faceDetector.SightingSimilarity", 0.6);
faceLogParameters.Set("faceDetector.FrameLag", 50);
faceLogParameters.Set("faceDetector.SpatialFilter", true);
faceLogParameters.Set("faceDetector.Motion", false);
faceLogParameters.Set("faceDetector.OSDFlags", 1);
PWatchlistOptions watchlistOptions;
watchlistOptions.SetTopN(1);
watchlistOptions.SetThreshold(0.75);
watchlistOptions.SetNormalise(false);
faceLogParameters.Set("faceDetector.WatchlistOptions", watchlistOptions);
faceLogParameters.Set("faceDetector.MaxFaceDetectorFR", -1);
faceLogParameters.Set("gpuId", -2); // Use optimized CPU models
PAnalytics::Create("FaceLog6", faceLogParameters, faceLog).OrDie("Creating simple face detector");
// Load calibration for the processor
fccalib.LoadFromFile(PPath::Join(SAMPLE_DIR, "calibrationMatrix.bin")).OrDie("Loading calibration matrix");
fcproc.SetCalibration(fccalib);
// Connect to feverCam
fcproc.HandleFeverCam(thermIP).OrDie("Connecting to feverCam");
// Connect to color camera
PInputVideoStream::Open(PString("rtsp://admin:%1@%2/stream0").Arg(colorPwd).Arg(colorIP), colorIvs)
.OrDie("Opening color stream");
PImage img;
PFrame colorFame, thermFrame;
// Detection will run as long as video streams are available
while(colorIvs.GetFrame(colorFame, 30000).LogErrorIfAny("retrieving color frame").Ok() && fcproc.IsActive()) {
PList faceEvents;
faceLog.Apply(colorFame, faceEvents).OrDie("processing color frame");
// Here we can detect black body should it be needed
PList bbodyCandidates;
if(fcproc.LocateBlackBody(colorFame, bbodyCandidates).LogErrorIfAny("Locating black body").Ok() &&
!bbodyCandidates.IsEmpty()) {
P_LOG_INFO << "Black body candidates: " << bbodyCandidates.ToString();
// At this point user could update automatically black body position in the camera
// To do this (very simple example assuming only one candidate is found):
//
// Retrieve the thermal video stream
// PInputVideoStream tivs;
// fcproc.GetTemperatureVideoSource(tivs).OrDie("Retrieving thermal video source");
// PRectanglef bbodyRect;
// bbodyCandidates.Get(0, bbodyRect).OrDie("Retrieving first black body candidate");
// tivs.Set("BLACKBODY_RECTANGLE", bbodyRect).OrDie("Setting black body position");
}
// Display synchronized temperature image
if(fcproc.GetThermalFrameAt(colorFame.GetTimestampUTC(), thermFrame).Ok())
thermFrame.Display("thermal", 0);
// Retrieve image that will be used for display of faces and temperature
img = colorFame.GetImageShared(PImage::E_RGB8U).Clone();
// Now process faces that we may have detected
for(int32 e = 0; e < faceEvents.Size(); ++e) {
PEvent evt;
faceEvents.Get(e, evt).OrDie("Retrieving event");
// We only process faces
if(evt.GetType() == "Face") {
// Extract the detection
const PProperties& eventProperties = evt.GetPayload();
PDetection detection;
if(eventProperties.Get("Detection", detection).LogIfError().Failed())
continue;
// We'll draw the image with the detection on it
// Estimate face temperature for this face
double temperature, confidence;
if(fcproc.EstimateFaceTemperature(detection, temperature, confidence)
.LogErrorIfAny("Estimating face temperature")) {
// We managed to get face temperature (so calibration was correct enough that we could match face to
// thermal equivalent)
P_LOG_INFO << "Face detected at " << detection.GetFrame().GetTimestampUTC().ToStringISOWithMs()
<< " has temperature " << temperature << "C with confidence " << confidence;
// we'll add the color information on top
PUtils::DrawLabel(img, detection, PString("T: %1C (conf: %2)").Arg(temperature).Arg(confidence),
PColour3i::Darkgray(), PColour3i::Red(), papillon::PUtils::E_TOP_CENTRE, 1., 0,
papillon::PColour3i::White(), papillon::PImage::E_FONT_HERSHEY_TRIPLEX);
}
}
}
// Display image with detection and temperature if available
img.Display("Detection", 0);
}
return 0;
}

Using the Library

  • Add PapillonFeverCam to your linker.
  • Add ${PAPILLON_INSTALL_DIR}/include/PapillonFeverCam to your include path.