diff --git a/4_video_sdks/camera_face_sdk/camera-face-sdk.iml b/4_video_sdks/camera_face_sdk/camera-face-sdk.iml new file mode 100644 index 00000000..9959c3f3 --- /dev/null +++ b/4_video_sdks/camera_face_sdk/camera-face-sdk.iml @@ -0,0 +1,216 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/4_video_sdks/camera_face_sdk/lib/aias-face-lib-0.1.0.jar b/4_video_sdks/camera_face_sdk/lib/aias-face-lib-0.1.0.jar deleted file mode 100644 index 65a056bb..00000000 Binary files a/4_video_sdks/camera_face_sdk/lib/aias-face-lib-0.1.0.jar and /dev/null differ diff --git a/4_video_sdks/camera_face_sdk/pom.xml b/4_video_sdks/camera_face_sdk/pom.xml index f49d999e..138b8a96 100644 --- a/4_video_sdks/camera_face_sdk/pom.xml +++ b/4_video_sdks/camera_face_sdk/pom.xml @@ -31,7 +31,7 @@ UTF-8 1.8 1.8 - 0.13.0 + 0.14.0 @@ -94,7 +94,7 @@ ai.djl.pytorch pytorch-native-auto - 1.9.0 + 1.9.1 @@ -113,13 +113,6 @@ paddlepaddle-model-zoo ${djl.version} - - aias - face-lib - 0.1.0 - system - ${project.basedir}/lib/aias-face-lib-0.1.0.jar - org.bytedeco javacv-platform diff --git a/4_video_sdks/camera_face_sdk/src/main/java/me/calvin/example/CameraFaceDetectionExample.java b/4_video_sdks/camera_face_sdk/src/main/java/me/calvin/example/CameraFaceDetectionExample.java index 2e02416a..6660ce21 100644 --- a/4_video_sdks/camera_face_sdk/src/main/java/me/calvin/example/CameraFaceDetectionExample.java +++ b/4_video_sdks/camera_face_sdk/src/main/java/me/calvin/example/CameraFaceDetectionExample.java @@ -11,7 +11,7 @@ import ai.djl.repository.zoo.Criteria; import ai.djl.repository.zoo.ModelZoo; import ai.djl.repository.zoo.ZooModel; import ai.djl.translate.TranslateException; -import me.aias.FaceDetection; +import me.calvin.example.utils.FaceDetection; import me.calvin.example.utils.OpenCVImageUtil; import org.bytedeco.javacv.CanvasFrame; import org.bytedeco.javacv.Frame; diff --git a/4_video_sdks/camera_face_sdk/src/main/java/me/calvin/example/utils/FaceDetection.java b/4_video_sdks/camera_face_sdk/src/main/java/me/calvin/example/utils/FaceDetection.java new file mode 100644 index 00000000..cee49203 --- /dev/null +++ b/4_video_sdks/camera_face_sdk/src/main/java/me/calvin/example/utils/FaceDetection.java @@ -0,0 +1,104 @@ +package me.calvin.example.utils; + +import ai.djl.Device; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.BoundingBox; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.modality.cv.output.Rectangle; +import ai.djl.modality.cv.util.NDImageUtils; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDList; +import ai.djl.ndarray.NDManager; +import ai.djl.ndarray.types.Shape; +import ai.djl.repository.zoo.Criteria; +import ai.djl.training.util.ProgressBar; +import ai.djl.translate.Batchifier; +import ai.djl.translate.Translator; +import ai.djl.translate.TranslatorContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public final class FaceDetection { + + private static final Logger logger = LoggerFactory.getLogger(FaceDetection.class); + + public FaceDetection() {} + + public Criteria criteria(float shrink, float threshold) { + Criteria criteria = + Criteria.builder() + .optEngine("PaddlePaddle") + .setTypes(Image.class, DetectedObjects.class) + .optModelUrls( + "https://aias-home.oss-cn-beijing.aliyuncs.com/models/face_mask/face_detection.zip") + // .optModelUrls("/Users/calvin/model/face_mask/pyramidbox_lite/") + // .optModelName("inference") + .optProgress(new ProgressBar()) + .optDevice(Device.cpu()) + .optTranslator(new FaceTranslator(shrink, threshold)) + .build(); + + return criteria; + } + + private final class FaceTranslator implements Translator { + + private float shrink; + private float threshold; + private List className; + + FaceTranslator(float shrink, float threshold) { + this.shrink = shrink; + this.threshold = threshold; + className = Arrays.asList("Not Face", "Face"); + } + + @Override + public DetectedObjects processOutput(TranslatorContext ctx, NDList list) { + return processImageOutput(list, className, threshold); + } + + @Override + public NDList processInput(TranslatorContext ctx, Image input) { + return processImageInput(ctx.getNDManager(), input, shrink); + } + + @Override + public Batchifier getBatchifier() { + return null; + } + + NDList processImageInput(NDManager manager, Image input, float shrink) { + NDArray array = input.toNDArray(manager); + Shape shape = array.getShape(); + array = + NDImageUtils.resize(array, (int) (shape.get(1) * shrink), (int) (shape.get(0) * shrink)); + array = array.transpose(2, 0, 1).flip(0); // HWC -> CHW BGR -> RGB + NDArray mean = manager.create(new float[] {104f, 117f, 123f}, new Shape(3, 1, 1)); + array = array.sub(mean).mul(0.007843f); // normalization + array = array.expandDims(0); // make batch dimension + return new NDList(array); + } + + DetectedObjects processImageOutput(NDList list, List className, float threshold) { + NDArray result = list.singletonOrThrow(); + float[] probabilities = result.get(":,1").toFloatArray(); + List names = new ArrayList<>(); + List prob = new ArrayList<>(); + List boxes = new ArrayList<>(); + for (int i = 0; i < probabilities.length; i++) { + if (probabilities[i] >= threshold) { + float[] array = result.get(i).toFloatArray(); + names.add(className.get((int) array[0])); + prob.add((double) probabilities[i]); + boxes.add(new Rectangle(array[2], array[3], array[4] - array[2], array[5] - array[3])); + } + } + return new DetectedObjects(names, prob, boxes); + } + } +} diff --git a/4_video_sdks/camera_facemask_sdk/camera-facemask-sdk.iml b/4_video_sdks/camera_facemask_sdk/camera-facemask-sdk.iml new file mode 100644 index 00000000..6278679a --- /dev/null +++ b/4_video_sdks/camera_facemask_sdk/camera-facemask-sdk.iml @@ -0,0 +1,216 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/4_video_sdks/camera_facemask_sdk/lib/aias-mask-lib-0.1.0.jar b/4_video_sdks/camera_facemask_sdk/lib/aias-mask-lib-0.1.0.jar deleted file mode 100644 index 071e0e43..00000000 Binary files a/4_video_sdks/camera_facemask_sdk/lib/aias-mask-lib-0.1.0.jar and /dev/null differ diff --git a/4_video_sdks/camera_facemask_sdk/pom.xml b/4_video_sdks/camera_facemask_sdk/pom.xml index f2714cf9..afcaa435 100644 --- a/4_video_sdks/camera_facemask_sdk/pom.xml +++ b/4_video_sdks/camera_facemask_sdk/pom.xml @@ -31,7 +31,7 @@ UTF-8 1.8 1.8 - 0.13.0 + 0.14.0 @@ -94,14 +94,13 @@ ai.djl.pytorch pytorch-native-auto - 1.9.0 + 1.9.1 ai.djl.paddlepaddle paddlepaddle-engine ${djl.version} - runtime ai.djl.paddlepaddle @@ -113,13 +112,6 @@ paddlepaddle-model-zoo ${djl.version} - - aias - mask-lib - 0.1.0 - system - ${project.basedir}/lib/aias-mask-lib-0.1.0.jar - org.bytedeco javacv-platform diff --git a/4_video_sdks/camera_facemask_sdk/src/main/java/me/aias/example/CameraFaceMaskDetectionExample.java b/4_video_sdks/camera_facemask_sdk/src/main/java/me/aias/example/CameraFaceMaskDetectionExample.java index c25e6056..d89734d4 100644 --- a/4_video_sdks/camera_facemask_sdk/src/main/java/me/aias/example/CameraFaceMaskDetectionExample.java +++ b/4_video_sdks/camera_facemask_sdk/src/main/java/me/aias/example/CameraFaceMaskDetectionExample.java @@ -12,8 +12,8 @@ import ai.djl.repository.zoo.Criteria; import ai.djl.repository.zoo.ModelZoo; import ai.djl.repository.zoo.ZooModel; import ai.djl.translate.TranslateException; -import me.aias.FaceDetection; -import me.aias.FaceMaskDetect; +import me.aias.example.utils.FaceDetection; +import me.aias.example.utils.FaceMaskDetect; import me.aias.example.utils.OpenCVImageUtil; import org.bytedeco.javacv.CanvasFrame; import org.bytedeco.javacv.Frame; @@ -151,7 +151,7 @@ public class CameraFaceMaskDetectionExample { if (squareBox[1] > height) squareBox[1] = height; if ((squareBox[0] + squareBox[2]) > width) squareBox[2] = width - squareBox[0]; if ((squareBox[1] + squareBox[2]) > height) squareBox[2] = height - squareBox[1]; - return img.getSubimage(squareBox[0], squareBox[1], squareBox[2], squareBox[2]); + return img.getSubImage(squareBox[0], squareBox[1], squareBox[2], squareBox[2]); // return img.getSubimage(squareBox[0], squareBox[1], squareBox[2], squareBox[3]); } } diff --git a/4_video_sdks/camera_facemask_sdk/src/main/java/me/aias/example/utils/FaceDetection.java b/4_video_sdks/camera_facemask_sdk/src/main/java/me/aias/example/utils/FaceDetection.java new file mode 100644 index 00000000..f5cfdd48 --- /dev/null +++ b/4_video_sdks/camera_facemask_sdk/src/main/java/me/aias/example/utils/FaceDetection.java @@ -0,0 +1,104 @@ +package me.aias.example.utils; + +import ai.djl.Device; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.BoundingBox; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.modality.cv.output.Rectangle; +import ai.djl.modality.cv.util.NDImageUtils; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDList; +import ai.djl.ndarray.NDManager; +import ai.djl.ndarray.types.Shape; +import ai.djl.repository.zoo.Criteria; +import ai.djl.training.util.ProgressBar; +import ai.djl.translate.Batchifier; +import ai.djl.translate.Translator; +import ai.djl.translate.TranslatorContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public final class FaceDetection { + + private static final Logger logger = LoggerFactory.getLogger(FaceDetection.class); + + public FaceDetection() {} + + public Criteria criteria(float shrink, float threshold) { + Criteria criteria = + Criteria.builder() + .optEngine("PaddlePaddle") + .setTypes(Image.class, DetectedObjects.class) + .optModelUrls( + "https://aias-home.oss-cn-beijing.aliyuncs.com/models/face_mask/face_detection.zip") + // .optModelUrls("/Users/calvin/model/face_mask/pyramidbox_lite/") + // .optModelName("inference") + .optProgress(new ProgressBar()) + .optDevice(Device.cpu()) + .optTranslator(new FaceTranslator(shrink, threshold)) + .build(); + + return criteria; + } + + private final class FaceTranslator implements Translator { + + private float shrink; + private float threshold; + private List className; + + FaceTranslator(float shrink, float threshold) { + this.shrink = shrink; + this.threshold = threshold; + className = Arrays.asList("Not Face", "Face"); + } + + @Override + public DetectedObjects processOutput(TranslatorContext ctx, NDList list) { + return processImageOutput(list, className, threshold); + } + + @Override + public NDList processInput(TranslatorContext ctx, Image input) { + return processImageInput(ctx.getNDManager(), input, shrink); + } + + @Override + public Batchifier getBatchifier() { + return null; + } + + NDList processImageInput(NDManager manager, Image input, float shrink) { + NDArray array = input.toNDArray(manager); + Shape shape = array.getShape(); + array = + NDImageUtils.resize(array, (int) (shape.get(1) * shrink), (int) (shape.get(0) * shrink)); + array = array.transpose(2, 0, 1).flip(0); // HWC -> CHW BGR -> RGB + NDArray mean = manager.create(new float[] {104f, 117f, 123f}, new Shape(3, 1, 1)); + array = array.sub(mean).mul(0.007843f); // normalization + array = array.expandDims(0); // make batch dimension + return new NDList(array); + } + + DetectedObjects processImageOutput(NDList list, List className, float threshold) { + NDArray result = list.singletonOrThrow(); + float[] probabilities = result.get(":,1").toFloatArray(); + List names = new ArrayList<>(); + List prob = new ArrayList<>(); + List boxes = new ArrayList<>(); + for (int i = 0; i < probabilities.length; i++) { + if (probabilities[i] >= threshold) { + float[] array = result.get(i).toFloatArray(); + names.add(className.get((int) array[0])); + prob.add((double) probabilities[i]); + boxes.add(new Rectangle(array[2], array[3], array[4] - array[2], array[5] - array[3])); + } + } + return new DetectedObjects(names, prob, boxes); + } + } +} diff --git a/4_video_sdks/camera_facemask_sdk/src/main/java/me/aias/example/utils/FaceMaskDetect.java b/4_video_sdks/camera_facemask_sdk/src/main/java/me/aias/example/utils/FaceMaskDetect.java new file mode 100644 index 00000000..bf6f9d92 --- /dev/null +++ b/4_video_sdks/camera_facemask_sdk/src/main/java/me/aias/example/utils/FaceMaskDetect.java @@ -0,0 +1,98 @@ +package me.aias.example.utils; + +import ai.djl.Device; +import ai.djl.inference.Predictor; +import ai.djl.modality.Classifications; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.BoundingBox; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.modality.cv.output.Rectangle; +import ai.djl.modality.cv.transform.Normalize; +import ai.djl.modality.cv.transform.Resize; +import ai.djl.modality.cv.transform.ToTensor; +import ai.djl.modality.cv.translator.ImageClassificationTranslator; +import ai.djl.repository.zoo.Criteria; +import ai.djl.training.util.ProgressBar; +import ai.djl.translate.TranslateException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +public final class FaceMaskDetect { + + private static final Logger logger = LoggerFactory.getLogger(FaceMaskDetect.class); + + public FaceMaskDetect() {} + + public DetectedObjects predict( + Predictor faceDetector, + Predictor classifier, + Image image) + throws TranslateException { + + DetectedObjects detections = faceDetector.predict(image); + List faces = detections.items(); + + List names = new ArrayList<>(); + List prob = new ArrayList<>(); + List rect = new ArrayList<>(); + for (DetectedObjects.DetectedObject face : faces) { + Image subImg = getSubImage(image, face.getBoundingBox()); + Classifications classifications = classifier.predict(subImg); + names.add(classifications.best().getClassName()); + prob.add(face.getProbability()); + rect.add(face.getBoundingBox()); + } + + return new DetectedObjects(names, prob, rect); + } + + public Criteria criteria() { + Criteria criteria = + Criteria.builder() + .optEngine("PaddlePaddle") + .setTypes(Image.class, Classifications.class) + .optTranslator( + ImageClassificationTranslator.builder() + .addTransform(new Resize(128, 128)) + .addTransform(new ToTensor()) // HWC -> CHW div(255) + .addTransform( + new Normalize( + new float[] {0.5f, 0.5f, 0.5f}, new float[] {1.0f, 1.0f, 1.0f})) + .addTransform(nd -> nd.flip(0)) // RGB -> GBR + .build()) + .optModelUrls( + "https://aias-home.oss-cn-beijing.aliyuncs.com/models/face_mask/face_mask.zip") + .optProgress(new ProgressBar()) + .optDevice(Device.cpu()) + .build(); + + return criteria; + } + + private int[] extendSquare( + double xmin, double ymin, double width, double height, double percentage) { + double centerx = xmin + width / 2; + double centery = ymin + height / 2; + double maxDist = Math.max(width / 2, height / 2) * (1 + percentage); + return new int[] {(int) (centerx - maxDist), (int) (centery - maxDist), (int) (2 * maxDist)}; + // return new int[] {(int) xmin, (int) ymin, (int) width, (int) height}; + } + + private Image getSubImage(Image img, BoundingBox box) { + Rectangle rect = box.getBounds(); + int width = img.getWidth(); + int height = img.getHeight(); + int[] squareBox = + extendSquare( + rect.getX() * width, + rect.getY() * height, + rect.getWidth() * width, + rect.getHeight() * height, + 0); // 0.18 + return img.getSubImage(squareBox[0], squareBox[1], squareBox[2], squareBox[2]); + // return img.getSubimage(squareBox[0], squareBox[1], squareBox[2], squareBox[3]); + } +} diff --git a/4_video_sdks/mp4_face_sdk/lib/aias-face-lib-0.1.0.jar b/4_video_sdks/mp4_face_sdk/lib/aias-face-lib-0.1.0.jar deleted file mode 100644 index 65a056bb..00000000 Binary files a/4_video_sdks/mp4_face_sdk/lib/aias-face-lib-0.1.0.jar and /dev/null differ diff --git a/4_video_sdks/mp4_face_sdk/mp4-face-sdk.iml b/4_video_sdks/mp4_face_sdk/mp4-face-sdk.iml new file mode 100644 index 00000000..aade2256 --- /dev/null +++ b/4_video_sdks/mp4_face_sdk/mp4-face-sdk.iml @@ -0,0 +1,225 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/4_video_sdks/mp4_face_sdk/pom.xml b/4_video_sdks/mp4_face_sdk/pom.xml index 6fc1fd77..662995ef 100644 --- a/4_video_sdks/mp4_face_sdk/pom.xml +++ b/4_video_sdks/mp4_face_sdk/pom.xml @@ -31,7 +31,7 @@ UTF-8 1.8 1.8 - 0.13.0 + 0.14.0 @@ -94,14 +94,13 @@ ai.djl.pytorch pytorch-native-auto - 1.9.0 + 1.9.1 ai.djl.paddlepaddle paddlepaddle-engine ${djl.version} - runtime ai.djl.paddlepaddle @@ -113,13 +112,6 @@ paddlepaddle-model-zoo ${djl.version} - - aias - face-lib - 0.1.0 - system - ${project.basedir}/lib/aias-face-lib-0.1.0.jar - org.bytedeco javacv-platform diff --git a/4_video_sdks/mp4_face_sdk/src/main/java/me/aias/example/MP4FaceDetectionExample.java b/4_video_sdks/mp4_face_sdk/src/main/java/me/aias/example/MP4FaceDetectionExample.java index 5246fd57..d4465385 100644 --- a/4_video_sdks/mp4_face_sdk/src/main/java/me/aias/example/MP4FaceDetectionExample.java +++ b/4_video_sdks/mp4_face_sdk/src/main/java/me/aias/example/MP4FaceDetectionExample.java @@ -11,7 +11,7 @@ import ai.djl.repository.zoo.Criteria; import ai.djl.repository.zoo.ModelZoo; import ai.djl.repository.zoo.ZooModel; import ai.djl.translate.TranslateException; -import me.aias.FaceDetection; +import me.aias.example.utils.FaceDetection; import me.aias.example.utils.OpenCVImageUtil; import org.bytedeco.javacv.CanvasFrame; import org.bytedeco.javacv.FFmpegFrameGrabber; diff --git a/4_video_sdks/mp4_face_sdk/src/main/java/me/aias/example/utils/FaceDetection.java b/4_video_sdks/mp4_face_sdk/src/main/java/me/aias/example/utils/FaceDetection.java new file mode 100644 index 00000000..f5cfdd48 --- /dev/null +++ b/4_video_sdks/mp4_face_sdk/src/main/java/me/aias/example/utils/FaceDetection.java @@ -0,0 +1,104 @@ +package me.aias.example.utils; + +import ai.djl.Device; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.BoundingBox; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.modality.cv.output.Rectangle; +import ai.djl.modality.cv.util.NDImageUtils; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDList; +import ai.djl.ndarray.NDManager; +import ai.djl.ndarray.types.Shape; +import ai.djl.repository.zoo.Criteria; +import ai.djl.training.util.ProgressBar; +import ai.djl.translate.Batchifier; +import ai.djl.translate.Translator; +import ai.djl.translate.TranslatorContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public final class FaceDetection { + + private static final Logger logger = LoggerFactory.getLogger(FaceDetection.class); + + public FaceDetection() {} + + public Criteria criteria(float shrink, float threshold) { + Criteria criteria = + Criteria.builder() + .optEngine("PaddlePaddle") + .setTypes(Image.class, DetectedObjects.class) + .optModelUrls( + "https://aias-home.oss-cn-beijing.aliyuncs.com/models/face_mask/face_detection.zip") + // .optModelUrls("/Users/calvin/model/face_mask/pyramidbox_lite/") + // .optModelName("inference") + .optProgress(new ProgressBar()) + .optDevice(Device.cpu()) + .optTranslator(new FaceTranslator(shrink, threshold)) + .build(); + + return criteria; + } + + private final class FaceTranslator implements Translator { + + private float shrink; + private float threshold; + private List className; + + FaceTranslator(float shrink, float threshold) { + this.shrink = shrink; + this.threshold = threshold; + className = Arrays.asList("Not Face", "Face"); + } + + @Override + public DetectedObjects processOutput(TranslatorContext ctx, NDList list) { + return processImageOutput(list, className, threshold); + } + + @Override + public NDList processInput(TranslatorContext ctx, Image input) { + return processImageInput(ctx.getNDManager(), input, shrink); + } + + @Override + public Batchifier getBatchifier() { + return null; + } + + NDList processImageInput(NDManager manager, Image input, float shrink) { + NDArray array = input.toNDArray(manager); + Shape shape = array.getShape(); + array = + NDImageUtils.resize(array, (int) (shape.get(1) * shrink), (int) (shape.get(0) * shrink)); + array = array.transpose(2, 0, 1).flip(0); // HWC -> CHW BGR -> RGB + NDArray mean = manager.create(new float[] {104f, 117f, 123f}, new Shape(3, 1, 1)); + array = array.sub(mean).mul(0.007843f); // normalization + array = array.expandDims(0); // make batch dimension + return new NDList(array); + } + + DetectedObjects processImageOutput(NDList list, List className, float threshold) { + NDArray result = list.singletonOrThrow(); + float[] probabilities = result.get(":,1").toFloatArray(); + List names = new ArrayList<>(); + List prob = new ArrayList<>(); + List boxes = new ArrayList<>(); + for (int i = 0; i < probabilities.length; i++) { + if (probabilities[i] >= threshold) { + float[] array = result.get(i).toFloatArray(); + names.add(className.get((int) array[0])); + prob.add((double) probabilities[i]); + boxes.add(new Rectangle(array[2], array[3], array[4] - array[2], array[5] - array[3])); + } + } + return new DetectedObjects(names, prob, boxes); + } + } +} diff --git a/4_video_sdks/mp4_facemask_sdk/lib/aias-mask-lib-0.1.0.jar b/4_video_sdks/mp4_facemask_sdk/lib/aias-mask-lib-0.1.0.jar deleted file mode 100644 index 071e0e43..00000000 Binary files a/4_video_sdks/mp4_facemask_sdk/lib/aias-mask-lib-0.1.0.jar and /dev/null differ diff --git a/4_video_sdks/mp4_facemask_sdk/mp4-facemask-sdk.iml b/4_video_sdks/mp4_facemask_sdk/mp4-facemask-sdk.iml new file mode 100644 index 00000000..6278679a --- /dev/null +++ b/4_video_sdks/mp4_facemask_sdk/mp4-facemask-sdk.iml @@ -0,0 +1,216 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/4_video_sdks/mp4_facemask_sdk/pom.xml b/4_video_sdks/mp4_facemask_sdk/pom.xml index 8b06f691..80ac3bce 100644 --- a/4_video_sdks/mp4_facemask_sdk/pom.xml +++ b/4_video_sdks/mp4_facemask_sdk/pom.xml @@ -31,7 +31,7 @@ UTF-8 1.8 1.8 - 0.13.0 + 0.14.0 @@ -94,14 +94,13 @@ ai.djl.pytorch pytorch-native-auto - 1.9.0 + 1.9.1 ai.djl.paddlepaddle paddlepaddle-engine ${djl.version} - runtime ai.djl.paddlepaddle @@ -118,13 +117,6 @@ javacv-platform 1.5.4 - - aias - mask-lib - 0.1.0 - system - ${project.basedir}/lib/aias-mask-lib-0.1.0.jar - org.testng testng diff --git a/4_video_sdks/mp4_facemask_sdk/src/main/java/me/aias/example/MP4FaceMaskDetectionExample.java b/4_video_sdks/mp4_facemask_sdk/src/main/java/me/aias/example/MP4FaceMaskDetectionExample.java index e8aec865..9c9e0f18 100644 --- a/4_video_sdks/mp4_facemask_sdk/src/main/java/me/aias/example/MP4FaceMaskDetectionExample.java +++ b/4_video_sdks/mp4_facemask_sdk/src/main/java/me/aias/example/MP4FaceMaskDetectionExample.java @@ -12,8 +12,8 @@ import ai.djl.repository.zoo.Criteria; import ai.djl.repository.zoo.ModelZoo; import ai.djl.repository.zoo.ZooModel; import ai.djl.translate.TranslateException; -import me.aias.FaceDetection; -import me.aias.FaceMaskDetect; +import me.aias.example.utils.FaceDetection; +import me.aias.example.utils.FaceMaskDetect; import me.aias.example.utils.OpenCVImageUtil; import org.bytedeco.javacv.CanvasFrame; import org.bytedeco.javacv.FFmpegFrameGrabber; @@ -156,7 +156,7 @@ public class MP4FaceMaskDetectionExample { if (squareBox[1] > height) squareBox[1] = height; if ((squareBox[0] + squareBox[2]) > width) squareBox[2] = width - squareBox[0]; if ((squareBox[1] + squareBox[2]) > height) squareBox[2] = height - squareBox[1]; - return img.getSubimage(squareBox[0], squareBox[1], squareBox[2], squareBox[2]); + return img.getSubImage(squareBox[0], squareBox[1], squareBox[2], squareBox[2]); // return img.getSubimage(squareBox[0], squareBox[1], squareBox[2], squareBox[3]); } } diff --git a/4_video_sdks/mp4_facemask_sdk/src/main/java/me/aias/example/utils/FaceDetection.java b/4_video_sdks/mp4_facemask_sdk/src/main/java/me/aias/example/utils/FaceDetection.java new file mode 100644 index 00000000..f5cfdd48 --- /dev/null +++ b/4_video_sdks/mp4_facemask_sdk/src/main/java/me/aias/example/utils/FaceDetection.java @@ -0,0 +1,104 @@ +package me.aias.example.utils; + +import ai.djl.Device; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.BoundingBox; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.modality.cv.output.Rectangle; +import ai.djl.modality.cv.util.NDImageUtils; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDList; +import ai.djl.ndarray.NDManager; +import ai.djl.ndarray.types.Shape; +import ai.djl.repository.zoo.Criteria; +import ai.djl.training.util.ProgressBar; +import ai.djl.translate.Batchifier; +import ai.djl.translate.Translator; +import ai.djl.translate.TranslatorContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public final class FaceDetection { + + private static final Logger logger = LoggerFactory.getLogger(FaceDetection.class); + + public FaceDetection() {} + + public Criteria criteria(float shrink, float threshold) { + Criteria criteria = + Criteria.builder() + .optEngine("PaddlePaddle") + .setTypes(Image.class, DetectedObjects.class) + .optModelUrls( + "https://aias-home.oss-cn-beijing.aliyuncs.com/models/face_mask/face_detection.zip") + // .optModelUrls("/Users/calvin/model/face_mask/pyramidbox_lite/") + // .optModelName("inference") + .optProgress(new ProgressBar()) + .optDevice(Device.cpu()) + .optTranslator(new FaceTranslator(shrink, threshold)) + .build(); + + return criteria; + } + + private final class FaceTranslator implements Translator { + + private float shrink; + private float threshold; + private List className; + + FaceTranslator(float shrink, float threshold) { + this.shrink = shrink; + this.threshold = threshold; + className = Arrays.asList("Not Face", "Face"); + } + + @Override + public DetectedObjects processOutput(TranslatorContext ctx, NDList list) { + return processImageOutput(list, className, threshold); + } + + @Override + public NDList processInput(TranslatorContext ctx, Image input) { + return processImageInput(ctx.getNDManager(), input, shrink); + } + + @Override + public Batchifier getBatchifier() { + return null; + } + + NDList processImageInput(NDManager manager, Image input, float shrink) { + NDArray array = input.toNDArray(manager); + Shape shape = array.getShape(); + array = + NDImageUtils.resize(array, (int) (shape.get(1) * shrink), (int) (shape.get(0) * shrink)); + array = array.transpose(2, 0, 1).flip(0); // HWC -> CHW BGR -> RGB + NDArray mean = manager.create(new float[] {104f, 117f, 123f}, new Shape(3, 1, 1)); + array = array.sub(mean).mul(0.007843f); // normalization + array = array.expandDims(0); // make batch dimension + return new NDList(array); + } + + DetectedObjects processImageOutput(NDList list, List className, float threshold) { + NDArray result = list.singletonOrThrow(); + float[] probabilities = result.get(":,1").toFloatArray(); + List names = new ArrayList<>(); + List prob = new ArrayList<>(); + List boxes = new ArrayList<>(); + for (int i = 0; i < probabilities.length; i++) { + if (probabilities[i] >= threshold) { + float[] array = result.get(i).toFloatArray(); + names.add(className.get((int) array[0])); + prob.add((double) probabilities[i]); + boxes.add(new Rectangle(array[2], array[3], array[4] - array[2], array[5] - array[3])); + } + } + return new DetectedObjects(names, prob, boxes); + } + } +} diff --git a/4_video_sdks/mp4_facemask_sdk/src/main/java/me/aias/example/utils/FaceMaskDetect.java b/4_video_sdks/mp4_facemask_sdk/src/main/java/me/aias/example/utils/FaceMaskDetect.java new file mode 100644 index 00000000..bf6f9d92 --- /dev/null +++ b/4_video_sdks/mp4_facemask_sdk/src/main/java/me/aias/example/utils/FaceMaskDetect.java @@ -0,0 +1,98 @@ +package me.aias.example.utils; + +import ai.djl.Device; +import ai.djl.inference.Predictor; +import ai.djl.modality.Classifications; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.BoundingBox; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.modality.cv.output.Rectangle; +import ai.djl.modality.cv.transform.Normalize; +import ai.djl.modality.cv.transform.Resize; +import ai.djl.modality.cv.transform.ToTensor; +import ai.djl.modality.cv.translator.ImageClassificationTranslator; +import ai.djl.repository.zoo.Criteria; +import ai.djl.training.util.ProgressBar; +import ai.djl.translate.TranslateException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +public final class FaceMaskDetect { + + private static final Logger logger = LoggerFactory.getLogger(FaceMaskDetect.class); + + public FaceMaskDetect() {} + + public DetectedObjects predict( + Predictor faceDetector, + Predictor classifier, + Image image) + throws TranslateException { + + DetectedObjects detections = faceDetector.predict(image); + List faces = detections.items(); + + List names = new ArrayList<>(); + List prob = new ArrayList<>(); + List rect = new ArrayList<>(); + for (DetectedObjects.DetectedObject face : faces) { + Image subImg = getSubImage(image, face.getBoundingBox()); + Classifications classifications = classifier.predict(subImg); + names.add(classifications.best().getClassName()); + prob.add(face.getProbability()); + rect.add(face.getBoundingBox()); + } + + return new DetectedObjects(names, prob, rect); + } + + public Criteria criteria() { + Criteria criteria = + Criteria.builder() + .optEngine("PaddlePaddle") + .setTypes(Image.class, Classifications.class) + .optTranslator( + ImageClassificationTranslator.builder() + .addTransform(new Resize(128, 128)) + .addTransform(new ToTensor()) // HWC -> CHW div(255) + .addTransform( + new Normalize( + new float[] {0.5f, 0.5f, 0.5f}, new float[] {1.0f, 1.0f, 1.0f})) + .addTransform(nd -> nd.flip(0)) // RGB -> GBR + .build()) + .optModelUrls( + "https://aias-home.oss-cn-beijing.aliyuncs.com/models/face_mask/face_mask.zip") + .optProgress(new ProgressBar()) + .optDevice(Device.cpu()) + .build(); + + return criteria; + } + + private int[] extendSquare( + double xmin, double ymin, double width, double height, double percentage) { + double centerx = xmin + width / 2; + double centery = ymin + height / 2; + double maxDist = Math.max(width / 2, height / 2) * (1 + percentage); + return new int[] {(int) (centerx - maxDist), (int) (centery - maxDist), (int) (2 * maxDist)}; + // return new int[] {(int) xmin, (int) ymin, (int) width, (int) height}; + } + + private Image getSubImage(Image img, BoundingBox box) { + Rectangle rect = box.getBounds(); + int width = img.getWidth(); + int height = img.getHeight(); + int[] squareBox = + extendSquare( + rect.getX() * width, + rect.getY() * height, + rect.getWidth() * width, + rect.getHeight() * height, + 0); // 0.18 + return img.getSubImage(squareBox[0], squareBox[1], squareBox[2], squareBox[2]); + // return img.getSubimage(squareBox[0], squareBox[1], squareBox[2], squareBox[3]); + } +} diff --git a/4_video_sdks/rtsp_face_sdk/lib/aias-face-lib-0.1.0.jar b/4_video_sdks/rtsp_face_sdk/lib/aias-face-lib-0.1.0.jar deleted file mode 100644 index 65a056bb..00000000 Binary files a/4_video_sdks/rtsp_face_sdk/lib/aias-face-lib-0.1.0.jar and /dev/null differ diff --git a/4_video_sdks/rtsp_face_sdk/pom.xml b/4_video_sdks/rtsp_face_sdk/pom.xml index d8e4edbf..a672c843 100644 --- a/4_video_sdks/rtsp_face_sdk/pom.xml +++ b/4_video_sdks/rtsp_face_sdk/pom.xml @@ -31,7 +31,7 @@ UTF-8 1.8 1.8 - 0.13.0 + 0.14.0 @@ -94,14 +94,13 @@ ai.djl.pytorch pytorch-native-auto - 1.9.0 + 1.9.1 ai.djl.paddlepaddle paddlepaddle-engine ${djl.version} - runtime ai.djl.paddlepaddle @@ -118,13 +117,6 @@ javacv-platform 1.5.4 - - aias - face-lib - 0.1.0 - system - ${project.basedir}/lib/aias-face-lib-0.1.0.jar - org.testng testng diff --git a/4_video_sdks/rtsp_face_sdk/rtsp-face-sdk.iml b/4_video_sdks/rtsp_face_sdk/rtsp-face-sdk.iml new file mode 100644 index 00000000..6278679a --- /dev/null +++ b/4_video_sdks/rtsp_face_sdk/rtsp-face-sdk.iml @@ -0,0 +1,216 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/4_video_sdks/rtsp_face_sdk/src/main/java/me/aias/example/RtspFaceDetectionExample.java b/4_video_sdks/rtsp_face_sdk/src/main/java/me/aias/example/RtspFaceDetectionExample.java index 520adb62..6e613dee 100644 --- a/4_video_sdks/rtsp_face_sdk/src/main/java/me/aias/example/RtspFaceDetectionExample.java +++ b/4_video_sdks/rtsp_face_sdk/src/main/java/me/aias/example/RtspFaceDetectionExample.java @@ -11,7 +11,7 @@ import ai.djl.repository.zoo.Criteria; import ai.djl.repository.zoo.ModelZoo; import ai.djl.repository.zoo.ZooModel; import ai.djl.translate.TranslateException; -import me.aias.FaceDetection; +import me.aias.example.utils.FaceDetection; import me.aias.example.utils.OpenCVImageUtil; import org.bytedeco.javacv.CanvasFrame; import org.bytedeco.javacv.FFmpegFrameGrabber; diff --git a/4_video_sdks/rtsp_face_sdk/src/main/java/me/aias/example/utils/FaceDetection.java b/4_video_sdks/rtsp_face_sdk/src/main/java/me/aias/example/utils/FaceDetection.java new file mode 100644 index 00000000..f5cfdd48 --- /dev/null +++ b/4_video_sdks/rtsp_face_sdk/src/main/java/me/aias/example/utils/FaceDetection.java @@ -0,0 +1,104 @@ +package me.aias.example.utils; + +import ai.djl.Device; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.BoundingBox; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.modality.cv.output.Rectangle; +import ai.djl.modality.cv.util.NDImageUtils; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDList; +import ai.djl.ndarray.NDManager; +import ai.djl.ndarray.types.Shape; +import ai.djl.repository.zoo.Criteria; +import ai.djl.training.util.ProgressBar; +import ai.djl.translate.Batchifier; +import ai.djl.translate.Translator; +import ai.djl.translate.TranslatorContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public final class FaceDetection { + + private static final Logger logger = LoggerFactory.getLogger(FaceDetection.class); + + public FaceDetection() {} + + public Criteria criteria(float shrink, float threshold) { + Criteria criteria = + Criteria.builder() + .optEngine("PaddlePaddle") + .setTypes(Image.class, DetectedObjects.class) + .optModelUrls( + "https://aias-home.oss-cn-beijing.aliyuncs.com/models/face_mask/face_detection.zip") + // .optModelUrls("/Users/calvin/model/face_mask/pyramidbox_lite/") + // .optModelName("inference") + .optProgress(new ProgressBar()) + .optDevice(Device.cpu()) + .optTranslator(new FaceTranslator(shrink, threshold)) + .build(); + + return criteria; + } + + private final class FaceTranslator implements Translator { + + private float shrink; + private float threshold; + private List className; + + FaceTranslator(float shrink, float threshold) { + this.shrink = shrink; + this.threshold = threshold; + className = Arrays.asList("Not Face", "Face"); + } + + @Override + public DetectedObjects processOutput(TranslatorContext ctx, NDList list) { + return processImageOutput(list, className, threshold); + } + + @Override + public NDList processInput(TranslatorContext ctx, Image input) { + return processImageInput(ctx.getNDManager(), input, shrink); + } + + @Override + public Batchifier getBatchifier() { + return null; + } + + NDList processImageInput(NDManager manager, Image input, float shrink) { + NDArray array = input.toNDArray(manager); + Shape shape = array.getShape(); + array = + NDImageUtils.resize(array, (int) (shape.get(1) * shrink), (int) (shape.get(0) * shrink)); + array = array.transpose(2, 0, 1).flip(0); // HWC -> CHW BGR -> RGB + NDArray mean = manager.create(new float[] {104f, 117f, 123f}, new Shape(3, 1, 1)); + array = array.sub(mean).mul(0.007843f); // normalization + array = array.expandDims(0); // make batch dimension + return new NDList(array); + } + + DetectedObjects processImageOutput(NDList list, List className, float threshold) { + NDArray result = list.singletonOrThrow(); + float[] probabilities = result.get(":,1").toFloatArray(); + List names = new ArrayList<>(); + List prob = new ArrayList<>(); + List boxes = new ArrayList<>(); + for (int i = 0; i < probabilities.length; i++) { + if (probabilities[i] >= threshold) { + float[] array = result.get(i).toFloatArray(); + names.add(className.get((int) array[0])); + prob.add((double) probabilities[i]); + boxes.add(new Rectangle(array[2], array[3], array[4] - array[2], array[5] - array[3])); + } + } + return new DetectedObjects(names, prob, boxes); + } + } +} diff --git a/4_video_sdks/rtsp_facemask_sdk/lib/aias-mask-lib-0.1.0.jar b/4_video_sdks/rtsp_facemask_sdk/lib/aias-mask-lib-0.1.0.jar deleted file mode 100644 index 071e0e43..00000000 Binary files a/4_video_sdks/rtsp_facemask_sdk/lib/aias-mask-lib-0.1.0.jar and /dev/null differ diff --git a/4_video_sdks/rtsp_facemask_sdk/pom.xml b/4_video_sdks/rtsp_facemask_sdk/pom.xml index 3dc6f2bd..696dda21 100644 --- a/4_video_sdks/rtsp_facemask_sdk/pom.xml +++ b/4_video_sdks/rtsp_facemask_sdk/pom.xml @@ -31,7 +31,7 @@ UTF-8 1.8 1.8 - 0.13.0 + 0.14.0 @@ -94,14 +94,13 @@ ai.djl.pytorch pytorch-native-auto - 1.9.0 + 1.9.1 ai.djl.paddlepaddle paddlepaddle-engine ${djl.version} - runtime ai.djl.paddlepaddle @@ -118,13 +117,6 @@ javacv-platform 1.5.4 - - aias - mask-lib - 0.1.0 - system - ${project.basedir}/lib/aias-mask-lib-0.1.0.jar - org.testng testng diff --git a/4_video_sdks/rtsp_facemask_sdk/rtsp-facemask-sdk.iml b/4_video_sdks/rtsp_facemask_sdk/rtsp-facemask-sdk.iml new file mode 100644 index 00000000..6278679a --- /dev/null +++ b/4_video_sdks/rtsp_facemask_sdk/rtsp-facemask-sdk.iml @@ -0,0 +1,216 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/4_video_sdks/rtsp_facemask_sdk/src/main/java/me/aias/example/RtspFaceMaskDetectionExample.java b/4_video_sdks/rtsp_facemask_sdk/src/main/java/me/aias/example/RtspFaceMaskDetectionExample.java index 98c7f83e..56d7c3fe 100644 --- a/4_video_sdks/rtsp_facemask_sdk/src/main/java/me/aias/example/RtspFaceMaskDetectionExample.java +++ b/4_video_sdks/rtsp_facemask_sdk/src/main/java/me/aias/example/RtspFaceMaskDetectionExample.java @@ -12,8 +12,8 @@ import ai.djl.repository.zoo.Criteria; import ai.djl.repository.zoo.ModelZoo; import ai.djl.repository.zoo.ZooModel; import ai.djl.translate.TranslateException; -import me.aias.FaceDetection; -import me.aias.FaceMaskDetect; +import me.aias.example.utils.FaceDetection; +import me.aias.example.utils.FaceMaskDetect; import me.aias.example.utils.OpenCVImageUtil; import org.bytedeco.javacv.CanvasFrame; import org.bytedeco.javacv.FFmpegFrameGrabber; @@ -191,7 +191,7 @@ public class RtspFaceMaskDetectionExample { if (squareBox[1] > height) squareBox[1] = height; if ((squareBox[0] + squareBox[2]) > width) squareBox[2] = width - squareBox[0]; if ((squareBox[1] + squareBox[2]) > height) squareBox[2] = height - squareBox[1]; - return img.getSubimage(squareBox[0], squareBox[1], squareBox[2], squareBox[2]); + return img.getSubImage(squareBox[0], squareBox[1], squareBox[2], squareBox[2]); // return img.getSubimage(squareBox[0], squareBox[1], squareBox[2], squareBox[3]); } } diff --git a/4_video_sdks/rtsp_facemask_sdk/src/main/java/me/aias/example/utils/FaceDetection.java b/4_video_sdks/rtsp_facemask_sdk/src/main/java/me/aias/example/utils/FaceDetection.java new file mode 100644 index 00000000..f5cfdd48 --- /dev/null +++ b/4_video_sdks/rtsp_facemask_sdk/src/main/java/me/aias/example/utils/FaceDetection.java @@ -0,0 +1,104 @@ +package me.aias.example.utils; + +import ai.djl.Device; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.BoundingBox; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.modality.cv.output.Rectangle; +import ai.djl.modality.cv.util.NDImageUtils; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDList; +import ai.djl.ndarray.NDManager; +import ai.djl.ndarray.types.Shape; +import ai.djl.repository.zoo.Criteria; +import ai.djl.training.util.ProgressBar; +import ai.djl.translate.Batchifier; +import ai.djl.translate.Translator; +import ai.djl.translate.TranslatorContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public final class FaceDetection { + + private static final Logger logger = LoggerFactory.getLogger(FaceDetection.class); + + public FaceDetection() {} + + public Criteria criteria(float shrink, float threshold) { + Criteria criteria = + Criteria.builder() + .optEngine("PaddlePaddle") + .setTypes(Image.class, DetectedObjects.class) + .optModelUrls( + "https://aias-home.oss-cn-beijing.aliyuncs.com/models/face_mask/face_detection.zip") + // .optModelUrls("/Users/calvin/model/face_mask/pyramidbox_lite/") + // .optModelName("inference") + .optProgress(new ProgressBar()) + .optDevice(Device.cpu()) + .optTranslator(new FaceTranslator(shrink, threshold)) + .build(); + + return criteria; + } + + private final class FaceTranslator implements Translator { + + private float shrink; + private float threshold; + private List className; + + FaceTranslator(float shrink, float threshold) { + this.shrink = shrink; + this.threshold = threshold; + className = Arrays.asList("Not Face", "Face"); + } + + @Override + public DetectedObjects processOutput(TranslatorContext ctx, NDList list) { + return processImageOutput(list, className, threshold); + } + + @Override + public NDList processInput(TranslatorContext ctx, Image input) { + return processImageInput(ctx.getNDManager(), input, shrink); + } + + @Override + public Batchifier getBatchifier() { + return null; + } + + NDList processImageInput(NDManager manager, Image input, float shrink) { + NDArray array = input.toNDArray(manager); + Shape shape = array.getShape(); + array = + NDImageUtils.resize(array, (int) (shape.get(1) * shrink), (int) (shape.get(0) * shrink)); + array = array.transpose(2, 0, 1).flip(0); // HWC -> CHW BGR -> RGB + NDArray mean = manager.create(new float[] {104f, 117f, 123f}, new Shape(3, 1, 1)); + array = array.sub(mean).mul(0.007843f); // normalization + array = array.expandDims(0); // make batch dimension + return new NDList(array); + } + + DetectedObjects processImageOutput(NDList list, List className, float threshold) { + NDArray result = list.singletonOrThrow(); + float[] probabilities = result.get(":,1").toFloatArray(); + List names = new ArrayList<>(); + List prob = new ArrayList<>(); + List boxes = new ArrayList<>(); + for (int i = 0; i < probabilities.length; i++) { + if (probabilities[i] >= threshold) { + float[] array = result.get(i).toFloatArray(); + names.add(className.get((int) array[0])); + prob.add((double) probabilities[i]); + boxes.add(new Rectangle(array[2], array[3], array[4] - array[2], array[5] - array[3])); + } + } + return new DetectedObjects(names, prob, boxes); + } + } +} diff --git a/4_video_sdks/rtsp_facemask_sdk/src/main/java/me/aias/example/utils/FaceMaskDetect.java b/4_video_sdks/rtsp_facemask_sdk/src/main/java/me/aias/example/utils/FaceMaskDetect.java new file mode 100644 index 00000000..bf6f9d92 --- /dev/null +++ b/4_video_sdks/rtsp_facemask_sdk/src/main/java/me/aias/example/utils/FaceMaskDetect.java @@ -0,0 +1,98 @@ +package me.aias.example.utils; + +import ai.djl.Device; +import ai.djl.inference.Predictor; +import ai.djl.modality.Classifications; +import ai.djl.modality.cv.Image; +import ai.djl.modality.cv.output.BoundingBox; +import ai.djl.modality.cv.output.DetectedObjects; +import ai.djl.modality.cv.output.Rectangle; +import ai.djl.modality.cv.transform.Normalize; +import ai.djl.modality.cv.transform.Resize; +import ai.djl.modality.cv.transform.ToTensor; +import ai.djl.modality.cv.translator.ImageClassificationTranslator; +import ai.djl.repository.zoo.Criteria; +import ai.djl.training.util.ProgressBar; +import ai.djl.translate.TranslateException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +public final class FaceMaskDetect { + + private static final Logger logger = LoggerFactory.getLogger(FaceMaskDetect.class); + + public FaceMaskDetect() {} + + public DetectedObjects predict( + Predictor faceDetector, + Predictor classifier, + Image image) + throws TranslateException { + + DetectedObjects detections = faceDetector.predict(image); + List faces = detections.items(); + + List names = new ArrayList<>(); + List prob = new ArrayList<>(); + List rect = new ArrayList<>(); + for (DetectedObjects.DetectedObject face : faces) { + Image subImg = getSubImage(image, face.getBoundingBox()); + Classifications classifications = classifier.predict(subImg); + names.add(classifications.best().getClassName()); + prob.add(face.getProbability()); + rect.add(face.getBoundingBox()); + } + + return new DetectedObjects(names, prob, rect); + } + + public Criteria criteria() { + Criteria criteria = + Criteria.builder() + .optEngine("PaddlePaddle") + .setTypes(Image.class, Classifications.class) + .optTranslator( + ImageClassificationTranslator.builder() + .addTransform(new Resize(128, 128)) + .addTransform(new ToTensor()) // HWC -> CHW div(255) + .addTransform( + new Normalize( + new float[] {0.5f, 0.5f, 0.5f}, new float[] {1.0f, 1.0f, 1.0f})) + .addTransform(nd -> nd.flip(0)) // RGB -> GBR + .build()) + .optModelUrls( + "https://aias-home.oss-cn-beijing.aliyuncs.com/models/face_mask/face_mask.zip") + .optProgress(new ProgressBar()) + .optDevice(Device.cpu()) + .build(); + + return criteria; + } + + private int[] extendSquare( + double xmin, double ymin, double width, double height, double percentage) { + double centerx = xmin + width / 2; + double centery = ymin + height / 2; + double maxDist = Math.max(width / 2, height / 2) * (1 + percentage); + return new int[] {(int) (centerx - maxDist), (int) (centery - maxDist), (int) (2 * maxDist)}; + // return new int[] {(int) xmin, (int) ymin, (int) width, (int) height}; + } + + private Image getSubImage(Image img, BoundingBox box) { + Rectangle rect = box.getBounds(); + int width = img.getWidth(); + int height = img.getHeight(); + int[] squareBox = + extendSquare( + rect.getX() * width, + rect.getY() * height, + rect.getWidth() * width, + rect.getHeight() * height, + 0); // 0.18 + return img.getSubImage(squareBox[0], squareBox[1], squareBox[2], squareBox[2]); + // return img.getSubimage(squareBox[0], squareBox[1], squareBox[2], squareBox[3]); + } +}