// Enable Mediapipe's "Hands" model
const handsfree = new Handsfree({hands: true})
// Enable plugins tagged with "browser"
handsfree.enablePlugins('browser')
// Start tracking
handsfree.start()
<aside> 🌞 Special thanks to: COSA, OSSTA, The STUDIO for Creative Inquiry, Glitch.com, Google PAIR, and you!
</aside>
https://www.youtube.com/watch?v=JywgoqmTr68
https://img.shields.io/github/stars/midiblocks/handsfree?style=social
<aside> ℹ️ Powered by: TensorFlow.js, MediaPipe, Jeeliz
</aside>
Gesture Mapper: Hands - Handsfree.js
<head>
<!-- Include Handsfree.js -->
<link rel="stylesheet" href="<https://unpkg.com/[email protected]/build/lib/assets/handsfree.css>" />
<script src="<https://unpkg.com/[email protected]/build/lib/handsfree.js>"></script>
</head>
<body>
<!-- Instantiate and start it after <body> -->
<script>
const handsfree = new Handsfree({hands: true})
handsfree.enablePlugins('browser')
handsfree.start()
</script>
</body>
# From your projects root
npm i handsfree
// Inside your app
import Handsfree from 'handsfree'
import 'handsfree/build/lib/assets/handsfree.css'
const handsfree = new Handsfree({hands: true})
handsfree.enablePlugins('browser')
handsfree.start()
<aside>
ℹ️ Each of the following models can be combined and reconfigured in real time with: handsfree.update({modelName: true}) Models can be combined, swapped in-and-out, and reconfigured in real time!
</aside>
Try it! Click the button below to read the rest of this page with hand tracking:
https://s3-us-west-2.amazonaws.com/secure.notion-static.com/756c4e86-e9d8-45ad-b312-8b672c6cfec9/hands-multi-scrolling_mp4_dvd.mp4
<table>
<thead>
<tr>
<th>Hand</th>
<th>Index [0]</th>
<th>Middle [1]</th>
<th>Ring [2]</th>
<th>Pinky [3]</th>
</tr>
</thead>
<tbody>
<tr>
<th>Left</th>
<td>
<div class="finger-pincher handsfree-hide-when-finger-pinched-0-0"></div>
<div class="finger-pincher handsfree-show-when-finger-pinched-0-0"></div>
</td>
<td>
<div class="finger-pincher handsfree-hide-when-finger-pinched-0-1"></div>
<div class="finger-pincher handsfree-show-when-finger-pinched-0-1"></div>
</td>
<td>
<div class="finger-pincher handsfree-hide-when-finger-pinched-0-2"></div>
<div class="finger-pincher handsfree-show-when-finger-pinched-0-2"></div>
</td>
<td>
<div class="finger-pincher handsfree-hide-when-finger-pinched-0-3"></div>
<div class="finger-pincher handsfree-show-when-finger-pinched-0-3"></div>
</td>
</tr>
<tr>
<th>Right</th>
<td>
<div class="finger-pincher handsfree-hide-when-finger-pinched-1-0"></div>
<div class="finger-pincher handsfree-show-when-finger-pinched-1-0"></div>
</td>
<td>
<div class="finger-pincher handsfree-hide-when-finger-pinched-1-1"></div>
<div class="finger-pincher handsfree-show-when-finger-pinched-1-1"></div>
</td>
<td>
<div class="finger-pincher handsfree-hide-when-finger-pinched-1-2"></div>
<div class="finger-pincher handsfree-show-when-finger-pinched-1-2"></div>
</td>
<td>
<div class="finger-pincher handsfree-hide-when-finger-pinched-1-3"></div>
<div class="finger-pincher handsfree-show-when-finger-pinched-1-3"></div>
</td>
</tr>
</tbody>
</table>
The circles above will turn red as you pinch the corresponding fingers. This feature is part of the Pincher plugin. The Pinchers plugin is tagged with “browser” and so is automatically enabled when you run handsfree.enablePlugins('browser')
You can use helper classes like .handsfree-show-when-loading and listen to events like handsfree-loading to help you quickly setup loading states for your app. For example, here’s a common “Go Handsfree” button pattern that is often used:
<span class="button">
<button class="handsfree-show-when-stopped handsfree-hide-when-loading" onclick="handsfree.start()">Go Handsfree</button>
<button class="handsfree-show-when-loading" disabled>...loading...</button>
<button class="handsfree-show-when-started handsfree-hide-when-loading" onclick="handsfree.stop()">Stop Handsfree</button>
</span>
handsfree.update({hands: true})
handsfree.enablePlugins('browser')
handsfree.start()
The following workflow demonstrates how to use all features of Handsfree.js.
// Let's enable face tracking with the default Face Pointer
const handsfree = new Handsfree({weboji: true})
handsfree.enablePlugins('browser')
// Now let's start things up
handsfree.start()
// Let's create a plugin called "logger"
// - Plugins run on every frame and is how you "plug in" to the main loop
// - "this" context is the plugin itself. In this case, handsfree.plugin.logger
handsfree.use('logger', data => {
console.log(data.weboji.morphs, data.weboji.rotation, data.weboji.pointer, data, this)
})
// Let's switch to hand tracking now. To demonstrate that you can do this live,
// let's create a plugin that switches to hand tracking when both eyebrows go up
handsfree.use('handTrackingSwitcher', ({weboji}) => {
if (weboji.state.browsUp) {
// Disable this plugin
// Same as handsfree.plugin.handTrackingSwitcher.disable()
this.disable()
// Turn off face tracking and enable hand tracking
handsfree.update({
weboji: false,
hands: true
})
}
})
// You can enable and disable any combination of models and plugins
handsfree.update({
// Disable weboji which is currently running
weboji: false,
// Start the pose model
pose: true,
// This is also how you configure (or pre-configure) a bunch of plugins at once
plugin: {
fingerPointer: {enabled: false},
faceScroll: {
vertScroll: {
scrollSpeed: 0.01
}
}
}
})
// Disable all plugins
handsfree.disablePlugins()
// Enable only the plugins for making music (not actually implemented yet)
handsfree.enablePlugins('music')
// Overwrite our logger to display the original model APIs
handsfree.plugin.logger.onFrame = (data) => {
console.log(handsfree.model.pose?.api, handsfree.model.weboji?.api, handsfree.model.pose?.api)
}
Handsfree.js is centered around a plugin system that lets you easily swap and combine functionality.
<aside> ℹ️ I'm still migrating other plugins from handsfree.dev/plugins
</aside>