From 9a822a2ed0717e62df78a07f6d5cc7a768166b3a Mon Sep 17 00:00:00 2001 From: QuinnHe Date: Wed, 27 Mar 2024 19:58:11 -0500 Subject: [PATCH] add body segmentation (#44) --- docs/reference/body-segmentation.md | 167 ++++++++++++++++++ docs/reference/bodypix.md | 99 ----------- docs/reference/bodypose.md | 2 +- docs/reference/facemesh.md | 2 +- docs/reference/handpose.md | 2 +- .../iframes/body-segmentation/index.html | 22 +++ .../iframes/body-segmentation/main.html | 24 +++ .../iframes/body-segmentation/script.js | 38 ++++ .../iframes/facemesh-keypoints/script.js | 2 +- .../iframes/handpose-keypoints/script.js | 2 +- .../iframes/pose-estimation/script.js | 2 +- docs/sidebar.md | 2 +- 12 files changed, 258 insertions(+), 106 deletions(-) create mode 100644 docs/reference/body-segmentation.md delete mode 100644 docs/reference/bodypix.md create mode 100644 docs/reference/iframes/body-segmentation/index.html create mode 100644 docs/reference/iframes/body-segmentation/main.html create mode 100644 docs/reference/iframes/body-segmentation/script.js diff --git a/docs/reference/body-segmentation.md b/docs/reference/body-segmentation.md new file mode 100644 index 0000000..9542c14 --- /dev/null +++ b/docs/reference/body-segmentation.md @@ -0,0 +1,167 @@ +# Body Segmentation + +
+ BodyPix Header Image of Harriet Tubman +
+ +## Description + +BodySegmentation divides an image input into the people and the background, and then further divide the people into different body parts. The model can detect multiple people at once and for each person, provides 24 body parts that describe important locations on the body. + +#### Key Features + +* Real-time person segmentation +* Real-time body part segmentation + +#### Output Example +Based on your options, the output can be a mask of the background, a mask of the person, or a mask of the body parts. Regardless of the scenario, the segmentation information deemed valuable is consistently stored within the mask property of the resulting object. + +```javascript +result.mask; +``` + +## Getting Started +Integrating Body Segmentation into your ml5.js projects is straightforward. Our documentation and user-friendly API will help you make the most of this combined model! + +### Demo + +[p5 Web Editor](iframes/body-segmentation ':include :type=iframe width=100% height=550px') + +### Quickstart +Before you start, let's create an empty project in the [p5 web editor](https://editor.p5js.org/). + +First of all, copy and paste the following code into your **index.html** file. If you are not familiar with the p5 web editor interface, you can find a guide on how to find your **index.html** file [here](/?id=try-ml5js-online-1). + +```html + +``` + +Then, add the code below to your **sketch.js** file: + +```js +let bodyPix; +let video; +let segmentation; + +let options = { + maskType: "parts", +}; + +function preload() { + bodyPix = ml5.bodySegmentation("BodyPix", options); +} + +function setup() { + createCanvas(640, 480); + // Create the video + video = createCapture(VIDEO); + video.size(width, height); + video.hide(); + + bodyPix.detectStart(video, gotResults); +} + +function draw() { + background(255); + image(video, 0, 0); + if (segmentation) { + image(segmentation, 0, 0, width, height); + } +} +// callback function for body segmentation +function gotResults(result) { + segmentation = result.mask; +} +``` +Alternatively, you can open [this example code](https://github.com/ml5js/ml5-next-gen/tree/main/examples/BodySegmentation-maskbodyparts) and try it yourself on p5.js web editor! + +### Additional Examples +* [BodySegmentation-maskbackground](https://github.com/ml5js/ml5-next-gen/tree/main/examples/BodySegmentation-maskbackground) +* [BodySegmentation-maskperson](https://github.com/ml5js/ml5-next-gen/tree/main/examples/BodySegmentation-maskperson) + + + +## Methods + +#### ml5.bodySegmentation() + +This method is used to initialize the bodySegmentation object. + +```javascript +const bodySegmentation = ml5.bodySegmentation(?modelName, ?options, ?callback); +``` + +**Parameters:** + +- **modelName**: OPTIONAL: A string specifying which model to use, "SelfieSegmentation" or "BodyPix". + +- **options**: OPTIONAL. An object to change the default configuration of the model. See the example options object: + + ```javascript + { + runtime: "mediapipe", // "mediapipe" or "tfjs" + modelType: "general", // "general" or "landscape" + maskType: :"background", //"background", "body", or "parts" (used to change the type of segmentation mask output) + } + ``` + + [More info on options for SelfieSegmentation model with mediaPipe runtime](https://github.com/tensorflow/tfjs-models/tree/master/body-segmentation/src/selfie_segmentation_mediapipe#create-a-detector). + + [More info on options for SelfieSegmentation model with tfjs runtime](https://github.com/tensorflow/tfjs-models/tree/master/body-segmentation/src/selfie_segmentation_tfjs#create-a-detector). + + [More infor on options for BodyPix model.](https://github.com/tensorflow/tfjs-models/blob/master/body-segmentation/src/body_pix/README.md#create-a-detector) + +- **callback(bodySegmentation, error)**: OPTIONAL. A function to run once the model has been loaded. Alternatively, call `ml5.bodySegmentation()` within the p5 `preload` function. + +**Returns:** +The bodySegmentation object. + +#### bodySegmentation.detectStart() + +This method repeatedly outputs segmentation masks on an image media through a callback function. + +```javascript +bodySegmentation.detectStart(media, callback); +``` + +**Parameters:** + +- **media**: An HTML or p5.js image, video, or canvas element to run the segmentation on. + +- **callback(output, error)**: A function to handle the output of `bodySegmentation.detectStart()`. Likely a function to do something with the segmented image. See below for the output passed into the callback function: + + ```javascript + { + mask: {},//A p5 Image object, can be directly passed into p5 image() function + maskImageData: {}//A ImageData object + } + ``` + +#### bodySegmentation.detectStop() + +This method can be called after a call to `bodySegmentation.detectStart` to stop the repeating pose estimation. + +```javascript +bodySegmentation.detectStop(); +``` + +#### bodySegmentation.detect() + +This method asynchronously outputs a single segmentation mask on an image media when called. + +```javascript +bodySegmentation.detect(media, ?callback); +``` + +**Parameters:** + +- **media**: An HTML or p5.js image, video, or canvas element to run the segmentation on. +- **callback(output, error)**: OPTIONAL. A callback function to handle the output of the estimation, see output example above. + +**Returns:** +A promise that resolves to the segmentation output. + +(footer needed) \ No newline at end of file diff --git a/docs/reference/bodypix.md b/docs/reference/bodypix.md deleted file mode 100644 index 0b49d98..0000000 --- a/docs/reference/bodypix.md +++ /dev/null @@ -1,99 +0,0 @@ -# BodyPix - - -
- BodyPix Header Image of Harriet Tubman -
- - -## Description -As written by the developers of BodyPix: - -"Bodypix is an open-source machine learning model which allows for person and body-part segmentation in the browser with TensorFlow.js. In computer vision, image segmentation refers to the technique of grouping pixels in an image into semantic areas typically to locate objects and boundaries. The BodyPix model is trained to do this for a person and twenty-four body parts (parts such as the left hand, front right lower leg, or back torso). In other words, BodyPix can classify the pixels of an image into two categories: 1) pixels that represent a person and 2) pixels that represent background. It can further classify pixels representing a person into any one of twenty-four body parts." - -## Quickstart - -```js -const bodypix = ml5.bodyPix(modelReady); - -function modelReady() { - // segment the image given - bodypix.segment(img, gotResults); -} - -function gotResults(error, result) { - if (error) { - console.log(error); - return; - } - // log the result - console.log(result.backgroundMask); -} -``` - -### Description - -As written by the developers of BodyPix: - -"Bodypix is an open-source machine learning model which allows for person and body-part segmentation in the browser with TensorFlow.js. In computer vision, image segmentation refers to the technique of grouping pixels in an image into semantic areas typically to locate objects and boundaries. The BodyPix model is trained to do this for a person and twenty-four body parts (parts such as the left hand, front right lower leg, or back torso). In other words, BodyPix can classify the pixels of an image into two categories: 1. pixels that represent a person and 2. pixels that represent background. It can further classify pixels representing a person into any one of twenty-four body parts." - -### Methods - -#### ml5.bodyPix() - -This method is used to initialize the bodyPix object. - -```javascript -const bodyPix = ml5.bodyPix(?video, ?options, ?callback); -``` - -**Parameters:** - -- **video**: OPTIONAL. An HTMLVideoElement or p5.Video to run the segmentation on. - -- **options**: OPTIONAL. An object to change the default configuration of the model. The default and available options are: - - ```javascript - { - architecture: "ResNet50", // "MobileNetV1" or "ResNet50" - multiplier: 1, // 0.5, 0.75 or 1 - outputStride: 16, // 8, 16, or 32 - quantBytes: 2, //1, 2 or 4 - } - ``` - - More info on options [here](https://github.com/tensorflow/tfjs-models/tree/master/body-segmentation/src/body_pix#create-a-detector). - -- **callback(bodyPix, error)**: OPTIONAL. A function to run once the model has been loaded. Alternatively, call `ml5.bodyPix()` within the p5 `preload` function. - -**Returns:** -The bodyPix object. - -#### bodyPix.segment() - -This method allows you to run segmentation on an image. - -```javascript -bodyPix.segment(?input, callback); -``` - -**Parameters:** - -- **input**: HTMLImageElement, HTMLVideoElement, ImageData, or HTMLCanvasElement. NOTE: Videos can be added through `ml5.bodyPix`. - -- **callback(output, error)**: A function to handle the output of `bodyPix.segment()`. Likely a function to do something with the segmented image. See below for the output passed into the callback function: - - ```javascript - { - backgroundMask, - bodyParts, - partMask, - personMask, - raw: { backgroundMask, partMask, personMask }, - segmentation: [{ mask, maskValueToLabel}, ...], - } - ``` - -### Examples - -TODO (link p5 web editor examples once uploaded) \ No newline at end of file diff --git a/docs/reference/bodypose.md b/docs/reference/bodypose.md index a361a38..7c271d8 100644 --- a/docs/reference/bodypose.md +++ b/docs/reference/bodypose.md @@ -166,7 +166,7 @@ let poses = []; function preload() { // Load the bodyPose model, default is MoveNet model - bodyPose = ml5.bodypose(); + bodyPose = ml5.bodyPose(); } function setup() { diff --git a/docs/reference/facemesh.md b/docs/reference/facemesh.md index e8e07e8..509f9b2 100644 --- a/docs/reference/facemesh.md +++ b/docs/reference/facemesh.md @@ -79,7 +79,7 @@ let options = { maxFaces: 1, refineLandmarks: false, flipHorizontal: false }; function preload() { // Load the faceMesh model - faceMesh = ml5.facemesh(options); + faceMesh = ml5.faceMesh(options); } function setup() { diff --git a/docs/reference/handpose.md b/docs/reference/handpose.md index 9b51513..5ef35c6 100644 --- a/docs/reference/handpose.md +++ b/docs/reference/handpose.md @@ -67,7 +67,7 @@ let hands = []; function preload() { // Load the handpose model - handpose = ml5.handpose(); + handpose = ml5.handPose(); } function setup() { diff --git a/docs/reference/iframes/body-segmentation/index.html b/docs/reference/iframes/body-segmentation/index.html new file mode 100644 index 0000000..dea5e3e --- /dev/null +++ b/docs/reference/iframes/body-segmentation/index.html @@ -0,0 +1,22 @@ + + + + + + + + p5.js iframe + + + + +
+

Body Segmentation Model

+

(Add description.) Ensure to enable the webcam.

+ +
+ + + \ No newline at end of file diff --git a/docs/reference/iframes/body-segmentation/main.html b/docs/reference/iframes/body-segmentation/main.html new file mode 100644 index 0000000..764d6aa --- /dev/null +++ b/docs/reference/iframes/body-segmentation/main.html @@ -0,0 +1,24 @@ + + + + + + + + + + + + + +
+
+
+ +
+ + + + \ No newline at end of file diff --git a/docs/reference/iframes/body-segmentation/script.js b/docs/reference/iframes/body-segmentation/script.js new file mode 100644 index 0000000..ad40c6f --- /dev/null +++ b/docs/reference/iframes/body-segmentation/script.js @@ -0,0 +1,38 @@ +// Copyright (c) 2020-2023 ml5 +// +// This software is released under the MIT License. +// https://opensource.org/licenses/MIT + +let bodyPix; +let video; +let segmentation; + +let options = { + maskType: "parts", +}; + +function preload() { + bodyPix = ml5.bodySegmentation("BodyPix", options); +} + +function setup() { + createCanvas(640, 480); + // Create the video + video = createCapture(VIDEO); + video.size(width, height); + video.hide(); + + bodyPix.detectStart(video, gotResults); +} + +function draw() { + background(255); + image(video, 0, 0); + if (segmentation) { + image(segmentation, 0, 0, width, height); + } +} +// callback function for body segmentation +function gotResults(result) { + segmentation = result.mask; +} \ No newline at end of file diff --git a/docs/reference/iframes/facemesh-keypoints/script.js b/docs/reference/iframes/facemesh-keypoints/script.js index 4d3165c..3d88996 100644 --- a/docs/reference/iframes/facemesh-keypoints/script.js +++ b/docs/reference/iframes/facemesh-keypoints/script.js @@ -10,7 +10,7 @@ let options = { maxFaces: 1, refineLandmarks: false, flipHorizontal: false }; function preload() { // Load the faceMesh model - faceMesh = ml5.facemesh(options); + faceMesh = ml5.faceMesh(options); } function setup() { diff --git a/docs/reference/iframes/handpose-keypoints/script.js b/docs/reference/iframes/handpose-keypoints/script.js index 3335da3..58bce17 100644 --- a/docs/reference/iframes/handpose-keypoints/script.js +++ b/docs/reference/iframes/handpose-keypoints/script.js @@ -9,7 +9,7 @@ let hands = []; function preload() { // Load the handpose model - handpose = ml5.handpose(); + handpose = ml5.handPose(); } function setup() { diff --git a/docs/reference/iframes/pose-estimation/script.js b/docs/reference/iframes/pose-estimation/script.js index ab8d861..0cbdfa1 100644 --- a/docs/reference/iframes/pose-estimation/script.js +++ b/docs/reference/iframes/pose-estimation/script.js @@ -9,7 +9,7 @@ let poses = []; function preload() { // Load the bodyPose model - bodyPose = ml5.bodypose(); + bodyPose = ml5.bodyPose(); } function setup() { diff --git a/docs/sidebar.md b/docs/sidebar.md index 719d1c1..2aeed40 100644 --- a/docs/sidebar.md +++ b/docs/sidebar.md @@ -11,7 +11,7 @@ * [Overview](/reference/overview.md) * **ml5 Models** 📦 * [Bodypose](/reference/bodypose.md) - + * [BodySegmentation](/reference/body-segmentation.md) * [Handpose](/reference/handpose.md) * [Facemesh](/reference/facemesh.md) * [ImageClassifier](/reference/image-classifier.md)