Difference between revisions of "MediaPipe for TouchDesigner"
(18 intermediate revisions by 2 users not shown) | |||
Line 96: | Line 96: | ||
To check if you can use MediaPipe in TouchDesigner, navigate to `Dialogs -> Textport and DATs`, import the MediaPipe module; `import mediapipe as mp` and list the components; `print(dir(mp.solutions))`. | To check if you can use MediaPipe in TouchDesigner, navigate to `Dialogs -> Textport and DATs`, import the MediaPipe module; `import mediapipe as mp` and list the components; `print(dir(mp.solutions))`. | ||
− | |||
− | + | '''what you also can do is use DAT Execute OP and copy and paste the code below.''' | |
− | + | '''Starter code:''' | |
− | + | <pre> | |
− | + | # me - this DAT | |
+ | # | ||
+ | # frame - the current frame | ||
+ | # state - True if the timeline is paused | ||
+ | # | ||
+ | # Make sure the corresponding toggle is enabled in the Execute DAT. | ||
− | + | def onStart(): | |
+ | import sys | ||
+ | |||
+ | pythonpath = "E:/Python/Lib/site-packages" | ||
+ | sys.path = [pythonpath] + sys.path | ||
+ | return | ||
− | + | def onCreate(): | |
− | + | return | |
− | + | def onExit(): | |
− | + | return | |
− | + | def onFrameStart(frame): | |
− | + | return | |
− | === | + | def onFrameEnd(frame): |
− | + | return | |
+ | |||
+ | def onPlayStateChange(state): | ||
+ | return | ||
+ | |||
+ | def onDeviceChange(): | ||
+ | return | ||
+ | |||
+ | def onProjectPreSave(): | ||
+ | return | ||
+ | |||
+ | def onProjectPostSave(): | ||
+ | return | ||
+ | |||
+ | </pre> | ||
+ | |||
+ | == MediaPipe in TouchDesigners== | ||
+ | |||
+ | |||
+ | |||
+ | ===1.faceDetect-CHOP=== | ||
+ | |||
+ | The code for script CHOP: | ||
+ | |||
+ | first, put a script CHOP on the empty network. and paste the code below: | ||
+ | |||
+ | |||
+ | <pre> | ||
+ | # me - this DAT | ||
+ | # scriptOp - the OP which is cooking | ||
+ | |||
+ | import numpy as np | ||
+ | import cv2 | ||
+ | import sys | ||
+ | import mediapipe as mp | ||
+ | |||
+ | mp_face = mp.solutions.face_detection | ||
+ | face_detection = mp_face.FaceDetection( | ||
+ | min_detection_confidence=0.7 | ||
+ | ) | ||
+ | |||
+ | # press 'Setup Parameters' in the OP to call this function to re-create the parameters. | ||
+ | def onSetupParameters(scriptOp): | ||
+ | page = scriptOp.appendCustomPage('Custom') | ||
+ | topPar = page.appendTOP('Face', label='Image with face') | ||
+ | return | ||
+ | |||
+ | # called whenever custom pulse parameter is pushed | ||
+ | def onPulse(par): | ||
+ | return | ||
+ | |||
+ | def onCook(scriptOp): | ||
+ | scriptOp.clear() | ||
+ | topRef = scriptOp.par.Face.eval() | ||
+ | |||
+ | num_faces = 0 | ||
+ | max_area = sys.float_info.min | ||
+ | width = 0 | ||
+ | height = 0 | ||
+ | xmin = 0.5 | ||
+ | ymin = 0.5 | ||
+ | lx = sys.float_info.max | ||
+ | ly = sys.float_info.max | ||
+ | rx = sys.float_info.max | ||
+ | ry = sys.float_info.max | ||
+ | |||
+ | if topRef: | ||
+ | img = topRef.numpyArray(delayed=True) | ||
+ | frame = cv2.cvtColor(img, cv2.COLOR_RGBA2RGB) | ||
+ | frame *= 255 | ||
+ | frame = frame.astype('uint8') | ||
+ | results = face_detection.process(frame) | ||
+ | |||
+ | if results.detections: | ||
+ | num_faces = len(results.detections) | ||
+ | |||
+ | for face in results.detections: | ||
+ | area = face.location_data.relative_bounding_box.width * face.location_data.relative_bounding_box.height | ||
+ | if area > max_area: | ||
+ | width = face.location_data.relative_bounding_box.width | ||
+ | height = face.location_data.relative_bounding_box.height | ||
+ | xmin = face.location_data.relative_bounding_box.xmin + width/2.0 | ||
+ | ymin = 1 - (face.location_data.relative_bounding_box.ymin + height/2.0) | ||
+ | lx = face.location_data.relative_keypoints[0].x | ||
+ | ly = 1 - face.location_data.relative_keypoints[0].y | ||
+ | rx = face.location_data.relative_keypoints[1].x | ||
+ | ry = 1 - face.location_data.relative_keypoints[1].y | ||
+ | |||
+ | max_area = area | ||
+ | |||
+ | tf = scriptOp.appendChan('face') | ||
+ | tw = scriptOp.appendChan('width') | ||
+ | th = scriptOp.appendChan('height') | ||
+ | tx = scriptOp.appendChan('tx') | ||
+ | ty = scriptOp.appendChan('ty') | ||
+ | leftx = scriptOp.appendChan('left_eye_x') | ||
+ | lefty = scriptOp.appendChan('left_eye_y') | ||
+ | rightx = scriptOp.appendChan('right_eye_x') | ||
+ | righty = scriptOp.appendChan('right_eye_y') | ||
+ | nosex = scriptOp.appendChan('nose_x') | ||
+ | nosey = scriptOp.appendChan('nose_y') | ||
+ | tf.vals = [num_faces] | ||
+ | tw.vals = [width] | ||
+ | th.vals = [height] | ||
+ | tx.vals = [xmin] | ||
+ | ty.vals = [ymin] | ||
+ | |||
+ | leftx.vals = [lx] | ||
+ | lefty.vals = [ly] | ||
+ | rightx.vals = [rx] | ||
+ | righty.vals = [ry] | ||
+ | |||
+ | |||
+ | scriptOp.rate = me.time.rate | ||
+ | |||
+ | return | ||
+ | </pre> | ||
+ | |||
+ | ===2.Hand Tracking CHOP-the index finger=== | ||
+ | |||
+ | <pre> | ||
+ | # me - this DAT | ||
+ | # scriptOp - the OP which is cooking | ||
+ | |||
+ | import numpy as np | ||
+ | import cv2 | ||
+ | import mediapipe as mp | ||
+ | |||
+ | mp_hands = mp.solutions.hands | ||
+ | hands = mp_hands.Hands( | ||
+ | max_num_hands=1, | ||
+ | min_detection_confidence=0.5, | ||
+ | min_tracking_confidence=0.5 | ||
+ | ) | ||
+ | |||
+ | # press 'Setup Parameters' in the OP to call this function to re-create the parameters. | ||
+ | def onSetupParameters(scriptOp): | ||
+ | page = scriptOp.appendCustomPage('Custom') | ||
+ | p = page.appendTOP('Image', label='Video image') | ||
+ | return | ||
+ | |||
+ | # called whenever custom pulse parameter is pushed | ||
+ | def onPulse(par): | ||
+ | return | ||
+ | |||
+ | def onCook(scriptOp): | ||
+ | scriptOp.clear() | ||
+ | input = scriptOp.par.Image.eval().numpyArray(delayed=True) | ||
+ | image = cv2.cvtColor(input, cv2.COLOR_RGBA2RGB) | ||
+ | image *= 255 | ||
+ | image = image.astype('uint8') | ||
+ | results = hands.process(image) | ||
+ | |||
+ | wrist = [] | ||
+ | index_tip = [] | ||
+ | num_hands = 0 | ||
+ | if results.multi_hand_landmarks: | ||
+ | num_hands += 1 | ||
+ | for hand in results.multi_hand_landmarks: | ||
+ | wrist.append(hand.landmark[0]) | ||
+ | index_tip.append(hand.landmark[8]) | ||
+ | |||
+ | tf = scriptOp.appendChan('hands') | ||
+ | tf.vals = [num_hands] | ||
+ | |||
+ | if len(wrist) > 0: | ||
+ | twx = scriptOp.appendChan('wrist:x') | ||
+ | twy = scriptOp.appendChan('wrist:y') | ||
+ | |||
+ | twx.vals = [wrist[0].x] | ||
+ | twy.vals = [wrist[0].y] | ||
+ | |||
+ | if len(index_tip) > 0: | ||
+ | tix = scriptOp.appendChan('index_tip:x') | ||
+ | tiy = scriptOp.appendChan('index_tip:y') | ||
+ | |||
+ | tix.vals = [index_tip[0].x] | ||
+ | tiy.vals = [index_tip[0].y] | ||
+ | |||
+ | scriptOp.rate = me.time.rate | ||
+ | |||
+ | return | ||
+ | </pre> | ||
+ | |||
+ | ===2.Hand Tracking CHOP-all fingers=== | ||
+ | |||
+ | <syntaxhighlight lang="python" line='line'> | ||
+ | |||
+ | </syntaxhighlight> | ||
+ | ===3.background replacement Selfie segmentation === | ||
+ | |||
+ | '''The TOP Script''' | ||
+ | |||
+ | |||
+ | <pre> | ||
+ | # me - this DAT | ||
+ | # scriptOp - the OP which is cooking | ||
+ | |||
+ | import numpy as np | ||
+ | import cv2 | ||
+ | import mediapipe as mp | ||
+ | |||
+ | mp_drawing = mp.solutions.drawing_utils | ||
+ | mp_pose = mp.solutions.pose | ||
+ | |||
+ | pose = mp_pose.Pose( | ||
+ | min_detection_confidence=0.5, | ||
+ | min_tracking_confidence=0.5, | ||
+ | enable_segmentation=True | ||
+ | ) | ||
+ | |||
+ | |||
+ | # press 'Setup Parameters' in the OP to call this function to re-create the parameters. | ||
+ | def onSetupParameters(scriptOp): | ||
+ | return | ||
+ | |||
+ | # called whenever custom pulse parameter is pushed | ||
+ | def onPulse(par): | ||
+ | return | ||
+ | |||
+ | |||
+ | def onCook(scriptOp): | ||
+ | input = scriptOp.inputs[0].numpyArray(delayed=True) | ||
+ | if input is not None: | ||
+ | image = cv2.cvtColor(input, cv2.COLOR_RGBA2RGB) | ||
+ | image *= 255 | ||
+ | image = image.astype('uint8') | ||
+ | results = pose.process(image) | ||
+ | |||
+ | if results.segmentation_mask is not None: | ||
+ | rgb = cv2.cvtColor(results.segmentation_mask, cv2.COLOR_GRAY2RGB) | ||
+ | rgb = rgb * 255 | ||
+ | rgb = rgb.astype(np.uint8) | ||
+ | scriptOp.copyNumpyArray(rgb) | ||
+ | else: | ||
+ | black = np.zeros(image.shape, dtype=np.uint8) | ||
+ | scriptOp.copyNumpyArray(black) | ||
+ | return | ||
+ | |||
+ | </pre> | ||
+ | |||
+ | ===4.posture tracking PoseCHOP === | ||
+ | |||
+ | <syntaxhighlight lang="python" line='line'> | ||
+ | |||
+ | </syntaxhighlight> | ||
+ | |||
+ | ==[[examples link:]]== | ||
+ | |||
+ | [[Category:TouchDesigner]] |
Latest revision as of 10:05, 6 February 2024
MediaPipe for TouchDesigner
On Apple Silicon (M1) using Rosetta2
We will install an emulated x86_64 brew using Rosetta2 alongside native Apple Silicon brew. This way we can install the by TouchDesigner expected python3.7 while also installing the MediaPipe module.
There is a MediaPipe module build for Apple Silicon (M1) machines found at [this Github issue](https://github.com/google/mediapipe/issues/3277), but I couldn't find a native Python 3.7 build for Apple Silicon to run in TouchDesigner.
Install brew x86_64 using Rosetta2
$ arch -x86_64 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
Make alias for x86_64 brew
Add this to rc file (.bashrc, .zshrc etc)
alias ibrew="arch -x86_64 /usr/local/bin/brew"
Install python3.7
$ ibrew install python@3.7
Alias for python@3.7 and pip3
Add to rc file
alias iPY37=/usr/local/opt/python@3.7/bin/python3
alias iPIP37=/usr/local/opt/python@3.7/bin/pip3
Now you can use `iPY37` to run x86_64 python@3.7 and `iPIP37` to install packages.
Install Mediapipe for python@3.7 x86_64
$ iPIP37 install mediapipe
Find python@3.7 x86_64 module path
Find the Python Module path of installed x86_64 python@3.7
$ ibrew --prefix
/usr/local
Append `/lib/python3.7/site-packages` to this path to get `/usr/local/lib/python3.7/site-packages`. This is the *Python 64-bit Module Path* you need to fill in TouchDesigner by going to `Preferences -> general -> Python 64-bit Module Path`
Restart TouchDesigner
Mediapipe should be installed! To verify the installation go to `Dialogs -> Textport and DATs`, import the MediaPipe module; `import mediapipe as mp` and list the components; `print(dir(mp.solutions))`
Windows
For windows we need to have a parallel copy of the same Python version on the harddrive. TouchDesigner expects us to work with Python3.7.x so grab this version from the [Python website](https://www.python.org/downloads/windows/). I used [Python 3.7.9](https://www.python.org/ftp/python/3.7.9/python-3.7.9-amd64.exe) for this tutorial.
Install excecutable
Double-click the downloaded executable file and install Python to the `DataStorage` drive `E:/` in a folder named `Python`
Installing MediaPipe
To install MediaPipe for our newly installed Python, open Windows PowerShell and run the following commands
$ python
...
Python 3.7.9
...
$ exit()
- Note: Check if the Python version corresponds with the Python you've installed in step 1.*
Install mediapipe using pip package manager:
$ pip install mediapipe
Open TouchDesigner starter
Download the [MediaPipe starter project](http://interactionstation.wdka.hro.nl/mediawiki/images/a/a5/Mp-starter-toe.zip), unzip it and double-click to open the project in TouchDesigner. If you installed Python3.7 in a different folder than `E:\Python\Lib\site-packages` you need to change the `pythonpath` variable within the `DAT Excecute` OP to the correct folder.
To check if you can use MediaPipe in TouchDesigner, navigate to `Dialogs -> Textport and DATs`, import the MediaPipe module; `import mediapipe as mp` and list the components; `print(dir(mp.solutions))`.
what you also can do is use DAT Execute OP and copy and paste the code below.
Starter code:
# me - this DAT # # frame - the current frame # state - True if the timeline is paused # # Make sure the corresponding toggle is enabled in the Execute DAT. def onStart(): import sys pythonpath = "E:/Python/Lib/site-packages" sys.path = [pythonpath] + sys.path return def onCreate(): return def onExit(): return def onFrameStart(frame): return def onFrameEnd(frame): return def onPlayStateChange(state): return def onDeviceChange(): return def onProjectPreSave(): return def onProjectPostSave(): return
MediaPipe in TouchDesigners
1.faceDetect-CHOP
The code for script CHOP:
first, put a script CHOP on the empty network. and paste the code below:
# me - this DAT # scriptOp - the OP which is cooking import numpy as np import cv2 import sys import mediapipe as mp mp_face = mp.solutions.face_detection face_detection = mp_face.FaceDetection( min_detection_confidence=0.7 ) # press 'Setup Parameters' in the OP to call this function to re-create the parameters. def onSetupParameters(scriptOp): page = scriptOp.appendCustomPage('Custom') topPar = page.appendTOP('Face', label='Image with face') return # called whenever custom pulse parameter is pushed def onPulse(par): return def onCook(scriptOp): scriptOp.clear() topRef = scriptOp.par.Face.eval() num_faces = 0 max_area = sys.float_info.min width = 0 height = 0 xmin = 0.5 ymin = 0.5 lx = sys.float_info.max ly = sys.float_info.max rx = sys.float_info.max ry = sys.float_info.max if topRef: img = topRef.numpyArray(delayed=True) frame = cv2.cvtColor(img, cv2.COLOR_RGBA2RGB) frame *= 255 frame = frame.astype('uint8') results = face_detection.process(frame) if results.detections: num_faces = len(results.detections) for face in results.detections: area = face.location_data.relative_bounding_box.width * face.location_data.relative_bounding_box.height if area > max_area: width = face.location_data.relative_bounding_box.width height = face.location_data.relative_bounding_box.height xmin = face.location_data.relative_bounding_box.xmin + width/2.0 ymin = 1 - (face.location_data.relative_bounding_box.ymin + height/2.0) lx = face.location_data.relative_keypoints[0].x ly = 1 - face.location_data.relative_keypoints[0].y rx = face.location_data.relative_keypoints[1].x ry = 1 - face.location_data.relative_keypoints[1].y max_area = area tf = scriptOp.appendChan('face') tw = scriptOp.appendChan('width') th = scriptOp.appendChan('height') tx = scriptOp.appendChan('tx') ty = scriptOp.appendChan('ty') leftx = scriptOp.appendChan('left_eye_x') lefty = scriptOp.appendChan('left_eye_y') rightx = scriptOp.appendChan('right_eye_x') righty = scriptOp.appendChan('right_eye_y') nosex = scriptOp.appendChan('nose_x') nosey = scriptOp.appendChan('nose_y') tf.vals = [num_faces] tw.vals = [width] th.vals = [height] tx.vals = [xmin] ty.vals = [ymin] leftx.vals = [lx] lefty.vals = [ly] rightx.vals = [rx] righty.vals = [ry] scriptOp.rate = me.time.rate return
2.Hand Tracking CHOP-the index finger
# me - this DAT # scriptOp - the OP which is cooking import numpy as np import cv2 import mediapipe as mp mp_hands = mp.solutions.hands hands = mp_hands.Hands( max_num_hands=1, min_detection_confidence=0.5, min_tracking_confidence=0.5 ) # press 'Setup Parameters' in the OP to call this function to re-create the parameters. def onSetupParameters(scriptOp): page = scriptOp.appendCustomPage('Custom') p = page.appendTOP('Image', label='Video image') return # called whenever custom pulse parameter is pushed def onPulse(par): return def onCook(scriptOp): scriptOp.clear() input = scriptOp.par.Image.eval().numpyArray(delayed=True) image = cv2.cvtColor(input, cv2.COLOR_RGBA2RGB) image *= 255 image = image.astype('uint8') results = hands.process(image) wrist = [] index_tip = [] num_hands = 0 if results.multi_hand_landmarks: num_hands += 1 for hand in results.multi_hand_landmarks: wrist.append(hand.landmark[0]) index_tip.append(hand.landmark[8]) tf = scriptOp.appendChan('hands') tf.vals = [num_hands] if len(wrist) > 0: twx = scriptOp.appendChan('wrist:x') twy = scriptOp.appendChan('wrist:y') twx.vals = [wrist[0].x] twy.vals = [wrist[0].y] if len(index_tip) > 0: tix = scriptOp.appendChan('index_tip:x') tiy = scriptOp.appendChan('index_tip:y') tix.vals = [index_tip[0].x] tiy.vals = [index_tip[0].y] scriptOp.rate = me.time.rate return
2.Hand Tracking CHOP-all fingers
3.background replacement Selfie segmentation
The TOP Script
# me - this DAT # scriptOp - the OP which is cooking import numpy as np import cv2 import mediapipe as mp mp_drawing = mp.solutions.drawing_utils mp_pose = mp.solutions.pose pose = mp_pose.Pose( min_detection_confidence=0.5, min_tracking_confidence=0.5, enable_segmentation=True ) # press 'Setup Parameters' in the OP to call this function to re-create the parameters. def onSetupParameters(scriptOp): return # called whenever custom pulse parameter is pushed def onPulse(par): return def onCook(scriptOp): input = scriptOp.inputs[0].numpyArray(delayed=True) if input is not None: image = cv2.cvtColor(input, cv2.COLOR_RGBA2RGB) image *= 255 image = image.astype('uint8') results = pose.process(image) if results.segmentation_mask is not None: rgb = cv2.cvtColor(results.segmentation_mask, cv2.COLOR_GRAY2RGB) rgb = rgb * 255 rgb = rgb.astype(np.uint8) scriptOp.copyNumpyArray(rgb) else: black = np.zeros(image.shape, dtype=np.uint8) scriptOp.copyNumpyArray(black) return