Android eye detection updated for OpenCV 2.4.6

Android eye detection updated for OpenCV 2.4.6

Posted on: 23.8.2013
By: Roman Hošek

With:
18 Comments

OpenCV samples were changed. In much more simple, light and elegant way. I get many emails with questions about problems with native camera and/or NDK setup, so I desided to write small update of previous sample in the new style. Now its only one class and
I removed all NDK setup, so it should be easier for beginners to run the project. The logic and principes are same, so for details how it works, please go through

Ok lets begin.

There was [java]mCamera = new VideoCapture(Highgui.CV_CAP_ANDROID+1);[/java] in SampleCvViewBase.java for switch from back to front camera.

Now its more simple, just set CameraIndex to 1 and as bonus, we enable build in Fps meter.

[java]

mOpenCvCameraView.setCameraIndex(1);

mOpenCvCameraView.enableFpsMeter();

[/java]

Create the „zooming“matrixes

[java]

private void CreateAuxiliaryMats() {

if (mGray.empty())

return;

int rows = mGray.rows();

int cols = mGray.cols();

if (mZoomWindow == null) {

mZoomWindow = mRgba.submat(rows / 2 + rows / 10, rows, cols / 2

+ cols / 10, cols);

mZoomWindow2 = mRgba.submat(0, rows / 2 – rows / 10, cols / 2

+ cols / 10, cols);

}

}

[/java]

creating eye template

[java]

private Mat get_template(CascadeClassifier clasificator, Rect area, int size) {

Mat template = new Mat();

Mat mROI = mGray.submat(area);

MatOfRect eyes = new MatOfRect();

Point iris = new Point();

Rect eye_template = new Rect();

clasificator.detectMultiScale(mROI, eyes, 1.15, 2,

Objdetect.CASCADE_FIND_BIGGEST_OBJECT

| Objdetect.CASCADE_SCALE_IMAGE, new Size(30, 30),

new Size());

Rect[] eyesArray = eyes.toArray();

for (int i = 0; i < eyesArray.length;) {

Rect e = eyesArray[i];

e.x = area.x + e.x;

e.y = area.y + e.y;

Rect eye_only_rectangle = new Rect((int) e.tl().x,

(int) (e.tl().y + e.height * 0.4), (int) e.width,

(int) (e.height * 0.6));

mROI = mGray.submat(eye_only_rectangle);

Mat vyrez = mRgba.submat(eye_only_rectangle);

Core.MinMaxLocResult mmG = Core.minMaxLoc(mROI);

Core.circle(vyrez, mmG.minLoc, 2, new Scalar(255, 255, 255, 255), 2);

iris.x = mmG.minLoc.x + eye_only_rectangle.x;

iris.y = mmG.minLoc.y + eye_only_rectangle.y;

eye_template = new Rect((int) iris.x – size / 2, (int) iris.y

- size / 2, size, size);

Core.rectangle(mRgba, eye_template.tl(), eye_template.br(),

new Scalar(255, 0, 0, 255), 2);

template = (mGray.submat(eye_template)).clone();

return template;

}

return template;

}

[/java]

and for matching eyes

[java]

private void match_eye(Rect area, Mat mTemplate, int type) {

Point matchLoc;

Mat mROI = mGray.submat(area);

int result_cols = mROI.cols() – mTemplate.cols() + 1;

int result_rows = mROI.rows() – mTemplate.rows() + 1;

// Check for bad template size

if (mTemplate.cols() == 0 || mTemplate.rows() == 0) {

return ;

}

Mat mResult = new Mat(result_cols, result_rows, CvType.CV_8U);

switch (type) {

case TM_SQDIFF:

Imgproc.matchTemplate(mROI, mTemplate, mResult, Imgproc.TM_SQDIFF);

break;

case TM_SQDIFF_NORMED:

Imgproc.matchTemplate(mROI, mTemplate, mResult,

Imgproc.TM_SQDIFF_NORMED);

break;

case TM_CCOEFF:

Imgproc.matchTemplate(mROI, mTemplate, mResult, Imgproc.TM_CCOEFF);

break;

case TM_CCOEFF_NORMED:

Imgproc.matchTemplate(mROI, mTemplate, mResult,

Imgproc.TM_CCOEFF_NORMED);

break;

case TM_CCORR:

Imgproc.matchTemplate(mROI, mTemplate, mResult, Imgproc.TM_CCORR);

break;

case TM_CCORR_NORMED:

Imgproc.matchTemplate(mROI, mTemplate, mResult,

Imgproc.TM_CCORR_NORMED);

break;

}

Core.MinMaxLocResult mmres = Core.minMaxLoc(mResult);

// there is difference in matching methods – best match is max/min value

if (type == TM_SQDIFF || type == TM_SQDIFF_NORMED) {

matchLoc = mmres.minLoc;

} else {

matchLoc = mmres.maxLoc;

}

Point matchLoc_tx = new Point(matchLoc.x + area.x, matchLoc.y + area.y);

Point matchLoc_ty = new Point(matchLoc.x + mTemplate.cols() + area.x,

matchLoc.y + mTemplate.rows() + area.y);

Core.rectangle(mRgba, matchLoc_tx, matchLoc_ty, new Scalar(255, 255, 0,

255));

Rect rec = new Rect(matchLoc_tx,matchLoc_ty);

}

[/java]

Layout is now inflated from xml instead of creating it by code (I dropped the VerticalSeekBar implementation to keep whole tutorial as one class)

[xml]

< RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"

xmlns:tools="http://schemas.android.com/tools"

android:layout_width="match_parent"

android:layout_height="match_parent" >

<org.opencv.android.JavaCameraView

android:id="@+id/fd_activity_surface_view"

android:layout_width="fill_parent"

android:layout_height="fill_parent" />

<Button

android:layout_width="wrap_content"

android:layout_height="wrap_content"

android:layout_alignParentLeft="true"

android:layout_centerVertical="true"

android:onClick="onRecreateClick"

android:text="Recreate" />

<TextView

android:id="@+id/method"

android:layout_width="wrap_content"

android:layout_height="wrap_content"

android:layout_above="@+id/seekbars"

android:layout_alignParentLeft="true"

android:layout_margin="5dp"

android:text="method"

android:textColor="@android:color/white"

android:textSize="10sp" />

<LinearLayout

android:id="@+id/seekbars"

android:layout_width="fill_parent"

android:layout_height="wrap_content"

android:layout_alignParentBottom="true"

android:orientation="horizontal"

>

<SeekBar

android:id="@+id/methodSeekBar"

android:layout_width="fill_parent"

android:layout_height="wrap_content"

android:max="5"

android:paddingLeft="5dip"

android:paddingRight="10dip"

android:progress="500" >

</SeekBar>

</LinearLayout>

</RelativeLayout>

[/xml]

so its pretty simple to add another widgets into layout.

And… its all, now working with OpenCV is faster and more intuitive:)

Final class:

[java]

public class FdActivity extends Activity implements CvCameraViewListener2 {

private static final String TAG = "OCVSample::Activity";

private static final Scalar FACE_RECT_COLOR = new Scalar(0, 255, 0, 255);

public static final int JAVA_DETECTOR = 0;

private static final int TM_SQDIFF = 0;

private static final int TM_SQDIFF_NORMED = 1;

private static final int TM_CCOEFF = 2;

private static final int TM_CCOEFF_NORMED = 3;

private static final int TM_CCORR = 4;

private static final int TM_CCORR_NORMED = 5;

private int learn_frames = 0;

private Mat teplateR;

private Mat teplateL;

int method = 0;

private MenuItem mItemFace50;

private MenuItem mItemFace40;

private MenuItem mItemFace30;

private MenuItem mItemFace20;

private MenuItem mItemType;

private Mat mRgba;

private Mat mGray;

// matrix for zooming

private Mat mZoomWindow;

private Mat mZoomWindow2;

private File mCascadeFile;

private CascadeClassifier mJavaDetector;

private CascadeClassifier mJavaDetectorEye;

private int mDetectorType = JAVA_DETECTOR;

private String[] mDetectorName;

private float mRelativeFaceSize = 0.2f;

private int mAbsoluteFaceSize = 0;

private CameraBridgeViewBase mOpenCvCameraView;

private SeekBar mMethodSeekbar;

private TextView mValue;

double xCenter = -1;

double yCenter = -1;

private BaseLoaderCallback mLoaderCallback = new BaseLoaderCallback(this) {

@Override

public void onManagerConnected(int status) {

switch (status) {

case LoaderCallbackInterface.SUCCESS: {

Log.i(TAG, "OpenCV loaded successfully");

try {

// load cascade file from application resources

InputStream is = getResources().openRawResource(

R.raw.lbpcascade_frontalface);

File cascadeDir = getDir("cascade", Context.MODE_PRIVATE);

mCascadeFile = new File(cascadeDir,

"lbpcascade_frontalface.xml");

FileOutputStream os = new FileOutputStream(mCascadeFile);

byte[] buffer = new byte[4096];

int bytesRead;

while ((bytesRead = is.read(buffer)) != -1) {

os.write(buffer, 0, bytesRead);

}

is.close();

os.close();

// ——————————— load left eye

// classificator ———————————–

InputStream iser = getResources().openRawResource(

R.raw.haarcascade_lefteye_2splits);

File cascadeDirER = getDir("cascadeER",

Context.MODE_PRIVATE);

File cascadeFileER = new File(cascadeDirER,

"haarcascade_eye_right.xml");

FileOutputStream oser = new FileOutputStream(cascadeFileER);

byte[] bufferER = new byte[4096];

int bytesReadER;

while ((bytesReadER = iser.read(bufferER)) != -1) {

oser.write(bufferER, 0, bytesReadER);

}

iser.close();

oser.close();

mJavaDetector = new CascadeClassifier(

mCascadeFile.getAbsolutePath());

if (mJavaDetector.empty()) {

Log.e(TAG, "Failed to load cascade classifier");

mJavaDetector = null;

} else

Log.i(TAG, "Loaded cascade classifier from "

+ mCascadeFile.getAbsolutePath());

mJavaDetectorEye = new CascadeClassifier(

cascadeFileER.getAbsolutePath());

if (mJavaDetectorEye.empty()) {

Log.e(TAG, "Failed to load cascade classifier");

mJavaDetectorEye = null;

} else

Log.i(TAG, "Loaded cascade classifier from "

+ mCascadeFile.getAbsolutePath());

cascadeDir.delete();

} catch (IOException e) {

e.printStackTrace();

Log.e(TAG, "Failed to load cascade. Exception thrown: " + e);

}

mOpenCvCameraView.setCameraIndex(1);

mOpenCvCameraView.enableFpsMeter();

mOpenCvCameraView.enableView();

}

break;

default: {

super.onManagerConnected(status);

}

break;

}

}

};

public FdActivity() {

mDetectorName = new String[2];

mDetectorName[JAVA_DETECTOR] = "Java";

Log.i(TAG, "Instantiated new " + this.getClass());

}

/** Called when the activity is first created. */

@Override

public void onCreate(Bundle savedInstanceState) {

Log.i(TAG, "called onCreate");

super.onCreate(savedInstanceState);

getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);

setContentView(R.layout.face_detect_surface_view);

mOpenCvCameraView = (CameraBridgeViewBase) findViewById(R.id.fd_activity_surface_view);

mOpenCvCameraView.setCvCameraViewListener(this);

mMethodSeekbar = (SeekBar) findViewById(R.id.methodSeekBar);

mValue = (TextView) findViewById(R.id.method);

mMethodSeekbar.setOnSeekBarChangeListener(new OnSeekBarChangeListener() {

@Override

public void onStopTrackingTouch(SeekBar seekBar) {

// TODO Auto-generated method stub

}

@Override

public void onStartTrackingTouch(SeekBar seekBar) {

// TODO Auto-generated method stub

}

@Override

public void onProgressChanged(SeekBar seekBar, int progress,

boolean fromUser) {

method = progress;

switch (method) {

case 0:

mValue.setText("TM_SQDIFF");

break;

case 1:

mValue.setText("TM_SQDIFF_NORMED");

break;

case 2:

mValue.setText("TM_CCOEFF");

break;

case 3:

mValue.setText("TM_CCOEFF_NORMED");

break;

case 4:

mValue.setText("TM_CCORR");

break;

case 5:

mValue.setText("TM_CCORR_NORMED");

break;

}

}

});

}

@Override

public void onPause() {

super.onPause();

if (mOpenCvCameraView != null)

mOpenCvCameraView.disableView();

}

@Override

public void onResume() {

super.onResume();

OpenCVLoader.initAsync(OpenCVLoader.OPENCV_VERSION_2_4_3, this,

mLoaderCallback);

}

public void onDestroy() {

super.onDestroy();

mOpenCvCameraView.disableView();

}

public void onCameraViewStarted(int width, int height) {

mGray = new Mat();

mRgba = new Mat();

}

public void onCameraViewStopped() {

mGray.release();

mRgba.release();

mZoomWindow.release();

mZoomWindow2.release();

}

public Mat onCameraFrame(CvCameraViewFrame inputFrame) {

mRgba = inputFrame.rgba();

mGray = inputFrame.gray();

if (mAbsoluteFaceSize == 0) {

int height = mGray.rows();

if (Math.round(height * mRelativeFaceSize) > 0) {

mAbsoluteFaceSize = Math.round(height * mRelativeFaceSize);

}

}

if (mZoomWindow == null || mZoomWindow2 == null)

CreateAuxiliaryMats();

MatOfRect faces = new MatOfRect();

if (mJavaDetector != null)

mJavaDetector.detectMultiScale(mGray, faces, 1.1, 2,

2, // TODO: objdetect.CV_HAAR_SCALE_IMAGE

new Size(mAbsoluteFaceSize, mAbsoluteFaceSize),

new Size());

Rect[] facesArray = faces.toArray();

for (int i = 0; i < facesArray.length; i++) {

Core.rectangle(mRgba, facesArray[i].tl(), facesArray[i].br(),

FACE_RECT_COLOR, 3);

xCenter = (facesArray[i].x + facesArray[i].width + facesArray[i].x) / 2;

yCenter = (facesArray[i].y + facesArray[i].y + facesArray[i].height) / 2;

Point center = new Point(xCenter, yCenter);

Core.circle(mRgba, center, 10, new Scalar(255, 0, 0, 255), 3);

Core.putText(mRgba, "[" + center.x + "," + center.y + "]",

new Point(center.x + 20, center.y + 20),

Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255,

255));

Rect r = facesArray[i];

// compute the eye area

Rect eyearea = new Rect(r.x + r.width / 8,

(int) (r.y + (r.height / 4.5)), r.width – 2 * r.width / 8,

(int) (r.height / 3.0));

// split it

Rect eyearea_right = new Rect(r.x + r.width / 16,

(int) (r.y + (r.height / 4.5)),

(r.width – 2 * r.width / 16) / 2, (int) (r.height / 3.0));

Rect eyearea_left = new Rect(r.x + r.width / 16

+ (r.width – 2 * r.width / 16) / 2,

(int) (r.y + (r.height / 4.5)),

(r.width – 2 * r.width / 16) / 2, (int) (r.height / 3.0));

// draw the area – mGray is working grayscale mat, if you want to

// see area in rgb preview, change mGray to mRgba

Core.rectangle(mRgba, eyearea_left.tl(), eyearea_left.br(),

new Scalar(255, 0, 0, 255), 2);

Core.rectangle(mRgba, eyearea_right.tl(), eyearea_right.br(),

new Scalar(255, 0, 0, 255), 2);

if (learn_frames < 5) {

teplateR = get_template(mJavaDetectorEye, eyearea_right, 24);

teplateL = get_template(mJavaDetectorEye, eyearea_left, 24);

learn_frames++;

} else {

// Learning finished, use the new templates for template

// matching

match_eye(eyearea_right, teplateR, method);

match_eye(eyearea_left, teplateL, method);

}

// cut eye areas and put them to zoom windows

Imgproc.resize(mRgba.submat(eyearea_left), mZoomWindow2,

mZoomWindow2.size());

Imgproc.resize(mRgba.submat(eyearea_right), mZoomWindow,

mZoomWindow.size());

}

return mRgba;

}

@Override

public boolean onCreateOptionsMenu(Menu menu) {

Log.i(TAG, "called onCreateOptionsMenu");

mItemFace50 = menu.add("Face size 50%");

mItemFace40 = menu.add("Face size 40%");

mItemFace30 = menu.add("Face size 30%");

mItemFace20 = menu.add("Face size 20%");

mItemType = menu.add(mDetectorName[mDetectorType]);

return true;

}

@Override

public boolean onOptionsItemSelected(MenuItem item) {

Log.i(TAG, "called onOptionsItemSelected; selected item: " + item);

if (item == mItemFace50)

setMinFaceSize(0.5f);

else if (item == mItemFace40)

setMinFaceSize(0.4f);

else if (item == mItemFace30)

setMinFaceSize(0.3f);

else if (item == mItemFace20)

setMinFaceSize(0.2f);

else if (item == mItemType) {

int tmpDetectorType = (mDetectorType + 1) % mDetectorName.length;

item.setTitle(mDetectorName[tmpDetectorType]);

}

return true;

}

private void setMinFaceSize(float faceSize) {

mRelativeFaceSize = faceSize;

mAbsoluteFaceSize = 0;

}

private void CreateAuxiliaryMats() {

if (mGray.empty())

return;

int rows = mGray.rows();

int cols = mGray.cols();

if (mZoomWindow == null) {

mZoomWindow = mRgba.submat(rows / 2 + rows / 10, rows, cols / 2

+ cols / 10, cols);

mZoomWindow2 = mRgba.submat(0, rows / 2 – rows / 10, cols / 2

+ cols / 10, cols);

}

}

private void match_eye(Rect area, Mat mTemplate, int type) {

Point matchLoc;

Mat mROI = mGray.submat(area);

int result_cols = mROI.cols() – mTemplate.cols() + 1;

int result_rows = mROI.rows() – mTemplate.rows() + 1;

// Check for bad template size

if (mTemplate.cols() == 0 || mTemplate.rows() == 0) {

return ;

}

Mat mResult = new Mat(result_cols, result_rows, CvType.CV_8U);

switch (type) {

case TM_SQDIFF:

Imgproc.matchTemplate(mROI, mTemplate, mResult, Imgproc.TM_SQDIFF);

break;

case TM_SQDIFF_NORMED:

Imgproc.matchTemplate(mROI, mTemplate, mResult,

Imgproc.TM_SQDIFF_NORMED);

break;

case TM_CCOEFF:

Imgproc.matchTemplate(mROI, mTemplate, mResult, Imgproc.TM_CCOEFF);

break;

case TM_CCOEFF_NORMED:

Imgproc.matchTemplate(mROI, mTemplate, mResult,

Imgproc.TM_CCOEFF_NORMED);

break;

case TM_CCORR:

Imgproc.matchTemplate(mROI, mTemplate, mResult, Imgproc.TM_CCORR);

break;

case TM_CCORR_NORMED:

Imgproc.matchTemplate(mROI, mTemplate, mResult,

Imgproc.TM_CCORR_NORMED);

break;

}

Core.MinMaxLocResult mmres = Core.minMaxLoc(mResult);

// there is difference in matching methods – best match is max/min value

if (type == TM_SQDIFF || type == TM_SQDIFF_NORMED) {

matchLoc = mmres.minLoc;

} else {

matchLoc = mmres.maxLoc;

}

Point matchLoc_tx = new Point(matchLoc.x + area.x, matchLoc.y + area.y);

Point matchLoc_ty = new Point(matchLoc.x + mTemplate.cols() + area.x,

matchLoc.y + mTemplate.rows() + area.y);

Core.rectangle(mRgba, matchLoc_tx, matchLoc_ty, new Scalar(255, 255, 0,

255));

Rect rec = new Rect(matchLoc_tx,matchLoc_ty);

}

private Mat get_template(CascadeClassifier clasificator, Rect area, int size) {

Mat template = new Mat();

Mat mROI = mGray.submat(area);

MatOfRect eyes = new MatOfRect();

Point iris = new Point();

Rect eye_template = new Rect();

clasificator.detectMultiScale(mROI, eyes, 1.15, 2,

Objdetect.CASCADE_FIND_BIGGEST_OBJECT

| Objdetect.CASCADE_SCALE_IMAGE, new Size(30, 30),

new Size());

Rect[] eyesArray = eyes.toArray();

for (int i = 0; i < eyesArray.length;) {

Rect e = eyesArray[i];

e.x = area.x + e.x;

e.y = area.y + e.y;

Rect eye_only_rectangle = new Rect((int) e.tl().x,

(int) (e.tl().y + e.height * 0.4), (int) e.width,

(int) (e.height * 0.6));

mROI = mGray.submat(eye_only_rectangle);

Mat vyrez = mRgba.submat(eye_only_rectangle);

Core.MinMaxLocResult mmG = Core.minMaxLoc(mROI);

Core.circle(vyrez, mmG.minLoc, 2, new Scalar(255, 255, 255, 255), 2);

iris.x = mmG.minLoc.x + eye_only_rectangle.x;

iris.y = mmG.minLoc.y + eye_only_rectangle.y;

eye_template = new Rect((int) iris.x – size / 2, (int) iris.y

- size / 2, size, size);

Core.rectangle(mRgba, eye_template.tl(), eye_template.br(),

new Scalar(255, 0, 0, 255), 2);

template = (mGray.submat(eye_template)).clone();

return template;

}

return template;

}

public void onRecreateClick(View v)

{

learn_frames = 0;

}

}

[/java]

Im getting lot of emails about problem with camera on your phone, before posting me email as bug, please check there, that is not OpenCV problem:
http://code.opencv.org/projects/opencv/issues?query_id=4