 TensorFlow.org에서 보기 |
 Google Colab에서 실행 |
 GitHub에서보기 |
 노트북 다운로드 |
 TF Hub 모델보기 |
이 Colab은 tfhub.dev/deepmind/i3d-kinetics-400/1 모듈을 사용하여 비디오 데이터에서 동작을 인식하는 방법을 보여줍니다. 비디오에서 동작을 감지하는 더 많은 모델은 여기에서 찾을 수 있습니다.
기본 모델은 Joao Carreira와 Andrew Zisserman의 "Quo Vadis, Action Recognition? A New Model and the Kinetics Dataset" 논문에 설명되어 있습니다. 이 논문은 2017년 5월 arXiv에 게시되었으며 CVPR 2017 컨퍼런스 논문으로 발표되었습니다. 소스 코드는 github에서 공개적으로 사용할 수 있습니다.
"Quo Vadis"는 비디오 분류를 위한 새로운 아키텍처인 Inflated 3D Convnet 또는 I3D를 도입했습니다. 이 아키텍처는 이러한 모델을 미세 조정하여 UCF101 및 HMDB51 데이터세트에서 최고의 결과를 얻었습니다. Kinetics에 사전 훈련된 I3D 모델도 CVPR 2017 Charades 챌린지에서 1위를 차지했습니다.
원래 모듈은 kinetics-400 데이터세트에서 훈련되었으며 약 400가지의 행동을 인식합니다. 이러한 행동의 레이블은 레이블 맵 파일에서 찾을 수 있습니다.
이 Colab에서는 UCF101 데이터세트에서 비디오 활동을 인식하는 데 이 모듈을 사용합니다.
설정
pip install -q imageiopip install -q opencv-pythonpip install -q git+https://github.com/tensorflow/docs
Import the necessary modules
# TensorFlow and TF-Hub modules.
from absl import logging
import tensorflow as tf
import tensorflow_hub as hub
from tensorflow_docs.vis import embed
logging.set_verbosity(logging.ERROR)
# Some modules to help with reading the UCF101 dataset.
import random
import re
import os
import tempfile
import ssl
import cv2
import numpy as np
# Some modules to display an animation using imageio.
import imageio
from IPython import display
from urllib import request # requires python3
2022-12-14 21:42:20.022962: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory 2022-12-14 21:42:20.023064: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory 2022-12-14 21:42:20.023074: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
Helper functions for the UCF101 dataset
# Utilities to fetch videos from UCF101 dataset
UCF_ROOT = "https://www.crcv.ucf.edu/THUMOS14/UCF101/UCF101/"
_VIDEO_LIST = None
_CACHE_DIR = tempfile.mkdtemp()
# As of July 2020, crcv.ucf.edu doesn't use a certificate accepted by the
# default Colab environment anymore.
unverified_context = ssl._create_unverified_context()
def list_ucf_videos():
"""Lists videos available in UCF101 dataset."""
global _VIDEO_LIST
if not _VIDEO_LIST:
index = request.urlopen(UCF_ROOT, context=unverified_context).read().decode("utf-8")
videos = re.findall("(v_[\w_]+\.avi)", index)
_VIDEO_LIST = sorted(set(videos))
return list(_VIDEO_LIST)
def fetch_ucf_video(video):
"""Fetchs a video and cache into local filesystem."""
cache_path = os.path.join(_CACHE_DIR, video)
if not os.path.exists(cache_path):
urlpath = request.urljoin(UCF_ROOT, video)
print("Fetching %s => %s" % (urlpath, cache_path))
data = request.urlopen(urlpath, context=unverified_context).read()
open(cache_path, "wb").write(data)
return cache_path
# Utilities to open video files using CV2
def crop_center_square(frame):
y, x = frame.shape[0:2]
min_dim = min(y, x)
start_x = (x // 2) - (min_dim // 2)
start_y = (y // 2) - (min_dim // 2)
return frame[start_y:start_y+min_dim,start_x:start_x+min_dim]
def load_video(path, max_frames=0, resize=(224, 224)):
cap = cv2.VideoCapture(path)
frames = []
try:
while True:
ret, frame = cap.read()
if not ret:
break
frame = crop_center_square(frame)
frame = cv2.resize(frame, resize)
frame = frame[:, :, [2, 1, 0]]
frames.append(frame)
if len(frames) == max_frames:
break
finally:
cap.release()
return np.array(frames) / 255.0
def to_gif(images):
converted_images = np.clip(images * 255, 0, 255).astype(np.uint8)
imageio.mimsave('./animation.gif', converted_images, fps=25)
return embed.embed_file('./animation.gif')
Get the kinetics-400 labels
# Get the kinetics-400 action labels from the GitHub repository.
KINETICS_URL = "https://raw.githubusercontent.com/deepmind/kinetics-i3d/master/data/label_map.txt"
with request.urlopen(KINETICS_URL) as obj:
labels = [line.decode("utf-8").strip() for line in obj.readlines()]
print("Found %d labels." % len(labels))
Found 400 labels.
UCF101 데이터세트 사용하기
# Get the list of videos in the dataset.
ucf_videos = list_ucf_videos()
categories = {}
for video in ucf_videos:
category = video[2:-12]
if category not in categories:
categories[category] = []
categories[category].append(video)
print("Found %d videos in %d categories." % (len(ucf_videos), len(categories)))
for category, sequences in categories.items():
summary = ", ".join(sequences[:2])
print("%-20s %4d videos (%s, ...)" % (category, len(sequences), summary))
Found 13320 videos in 101 categories. ApplyEyeMakeup 145 videos (v_ApplyEyeMakeup_g01_c01.avi, v_ApplyEyeMakeup_g01_c02.avi, ...) ApplyLipstick 114 videos (v_ApplyLipstick_g01_c01.avi, v_ApplyLipstick_g01_c02.avi, ...) Archery 145 videos (v_Archery_g01_c01.avi, v_Archery_g01_c02.avi, ...) BabyCrawling 132 videos (v_BabyCrawling_g01_c01.avi, v_BabyCrawling_g01_c02.avi, ...) BalanceBeam 108 videos (v_BalanceBeam_g01_c01.avi, v_BalanceBeam_g01_c02.avi, ...) BandMarching 155 videos (v_BandMarching_g01_c01.avi, v_BandMarching_g01_c02.avi, ...) BaseballPitch 150 videos (v_BaseballPitch_g01_c01.avi, v_BaseballPitch_g01_c02.avi, ...) BasketballDunk 131 videos (v_BasketballDunk_g01_c01.avi, v_BasketballDunk_g01_c02.avi, ...) Basketball 134 videos (v_Basketball_g01_c01.avi, v_Basketball_g01_c02.avi, ...) BenchPress 160 videos (v_BenchPress_g01_c01.avi, v_BenchPress_g01_c02.avi, ...) Biking 134 videos (v_Biking_g01_c01.avi, v_Biking_g01_c02.avi, ...) Billiards 150 videos (v_Billiards_g01_c01.avi, v_Billiards_g01_c02.avi, ...) BlowDryHair 131 videos (v_BlowDryHair_g01_c01.avi, v_BlowDryHair_g01_c02.avi, ...) BlowingCandles 109 videos (v_BlowingCandles_g01_c01.avi, v_BlowingCandles_g01_c02.avi, ...) BodyWeightSquats 112 videos (v_BodyWeightSquats_g01_c01.avi, v_BodyWeightSquats_g01_c02.avi, ...) Bowling 155 videos (v_Bowling_g01_c01.avi, v_Bowling_g01_c02.avi, ...) BoxingPunchingBag 163 videos (v_BoxingPunchingBag_g01_c01.avi, v_BoxingPunchingBag_g01_c02.avi, ...) BoxingSpeedBag 134 videos (v_BoxingSpeedBag_g01_c01.avi, v_BoxingSpeedBag_g01_c02.avi, ...) BreastStroke 101 videos (v_BreastStroke_g01_c01.avi, v_BreastStroke_g01_c02.avi, ...) BrushingTeeth 131 videos (v_BrushingTeeth_g01_c01.avi, v_BrushingTeeth_g01_c02.avi, ...) CleanAndJerk 112 videos (v_CleanAndJerk_g01_c01.avi, v_CleanAndJerk_g01_c02.avi, ...) CliffDiving 138 videos (v_CliffDiving_g01_c01.avi, v_CliffDiving_g01_c02.avi, ...) CricketBowling 139 videos (v_CricketBowling_g01_c01.avi, v_CricketBowling_g01_c02.avi, ...) CricketShot 167 videos (v_CricketShot_g01_c01.avi, v_CricketShot_g01_c02.avi, ...) CuttingInKitchen 110 videos (v_CuttingInKitchen_g01_c01.avi, v_CuttingInKitchen_g01_c02.avi, ...) Diving 150 videos (v_Diving_g01_c01.avi, v_Diving_g01_c02.avi, ...) Drumming 161 videos (v_Drumming_g01_c01.avi, v_Drumming_g01_c02.avi, ...) Fencing 111 videos (v_Fencing_g01_c01.avi, v_Fencing_g01_c02.avi, ...) FieldHockeyPenalty 126 videos (v_FieldHockeyPenalty_g01_c01.avi, v_FieldHockeyPenalty_g01_c02.avi, ...) FloorGymnastics 125 videos (v_FloorGymnastics_g01_c01.avi, v_FloorGymnastics_g01_c02.avi, ...) FrisbeeCatch 126 videos (v_FrisbeeCatch_g01_c01.avi, v_FrisbeeCatch_g01_c02.avi, ...) FrontCrawl 137 videos (v_FrontCrawl_g01_c01.avi, v_FrontCrawl_g01_c02.avi, ...) GolfSwing 139 videos (v_GolfSwing_g01_c01.avi, v_GolfSwing_g01_c02.avi, ...) Haircut 130 videos (v_Haircut_g01_c01.avi, v_Haircut_g01_c02.avi, ...) HammerThrow 150 videos (v_HammerThrow_g01_c01.avi, v_HammerThrow_g01_c02.avi, ...) Hammering 140 videos (v_Hammering_g01_c01.avi, v_Hammering_g01_c02.avi, ...) HandstandPushups 128 videos (v_HandstandPushups_g01_c01.avi, v_HandstandPushups_g01_c02.avi, ...) HandstandWalking 111 videos (v_HandstandWalking_g01_c01.avi, v_HandstandWalking_g01_c02.avi, ...) HeadMassage 147 videos (v_HeadMassage_g01_c01.avi, v_HeadMassage_g01_c02.avi, ...) HighJump 123 videos (v_HighJump_g01_c01.avi, v_HighJump_g01_c02.avi, ...) HorseRace 124 videos (v_HorseRace_g01_c01.avi, v_HorseRace_g01_c02.avi, ...) HorseRiding 164 videos (v_HorseRiding_g01_c01.avi, v_HorseRiding_g01_c02.avi, ...) HulaHoop 125 videos (v_HulaHoop_g01_c01.avi, v_HulaHoop_g01_c02.avi, ...) IceDancing 158 videos (v_IceDancing_g01_c01.avi, v_IceDancing_g01_c02.avi, ...) JavelinThrow 117 videos (v_JavelinThrow_g01_c01.avi, v_JavelinThrow_g01_c02.avi, ...) JugglingBalls 121 videos (v_JugglingBalls_g01_c01.avi, v_JugglingBalls_g01_c02.avi, ...) JumpRope 144 videos (v_JumpRope_g01_c01.avi, v_JumpRope_g01_c02.avi, ...) JumpingJack 123 videos (v_JumpingJack_g01_c01.avi, v_JumpingJack_g01_c02.avi, ...) Kayaking 141 videos (v_Kayaking_g01_c01.avi, v_Kayaking_g01_c02.avi, ...) Knitting 123 videos (v_Knitting_g01_c01.avi, v_Knitting_g01_c02.avi, ...) LongJump 131 videos (v_LongJump_g01_c01.avi, v_LongJump_g01_c02.avi, ...) Lunges 127 videos (v_Lunges_g01_c01.avi, v_Lunges_g01_c02.avi, ...) MilitaryParade 125 videos (v_MilitaryParade_g01_c01.avi, v_MilitaryParade_g01_c02.avi, ...) Mixing 136 videos (v_Mixing_g01_c01.avi, v_Mixing_g01_c02.avi, ...) MoppingFloor 110 videos (v_MoppingFloor_g01_c01.avi, v_MoppingFloor_g01_c02.avi, ...) Nunchucks 132 videos (v_Nunchucks_g01_c01.avi, v_Nunchucks_g01_c02.avi, ...) ParallelBars 114 videos (v_ParallelBars_g01_c01.avi, v_ParallelBars_g01_c02.avi, ...) PizzaTossing 113 videos (v_PizzaTossing_g01_c01.avi, v_PizzaTossing_g01_c02.avi, ...) PlayingCello 164 videos (v_PlayingCello_g01_c01.avi, v_PlayingCello_g01_c02.avi, ...) PlayingDaf 151 videos (v_PlayingDaf_g01_c01.avi, v_PlayingDaf_g01_c02.avi, ...) PlayingDhol 164 videos (v_PlayingDhol_g01_c01.avi, v_PlayingDhol_g01_c02.avi, ...) PlayingFlute 155 videos (v_PlayingFlute_g01_c01.avi, v_PlayingFlute_g01_c02.avi, ...) PlayingGuitar 160 videos (v_PlayingGuitar_g01_c01.avi, v_PlayingGuitar_g01_c02.avi, ...) PlayingPiano 105 videos (v_PlayingPiano_g01_c01.avi, v_PlayingPiano_g01_c02.avi, ...) PlayingSitar 157 videos (v_PlayingSitar_g01_c01.avi, v_PlayingSitar_g01_c02.avi, ...) PlayingTabla 111 videos (v_PlayingTabla_g01_c01.avi, v_PlayingTabla_g01_c02.avi, ...) PlayingViolin 100 videos (v_PlayingViolin_g01_c01.avi, v_PlayingViolin_g01_c02.avi, ...) PoleVault 149 videos (v_PoleVault_g01_c01.avi, v_PoleVault_g01_c02.avi, ...) PommelHorse 123 videos (v_PommelHorse_g01_c01.avi, v_PommelHorse_g01_c02.avi, ...) PullUps 100 videos (v_PullUps_g01_c01.avi, v_PullUps_g01_c02.avi, ...) Punch 160 videos (v_Punch_g01_c01.avi, v_Punch_g01_c02.avi, ...) PushUps 102 videos (v_PushUps_g01_c01.avi, v_PushUps_g01_c02.avi, ...) Rafting 111 videos (v_Rafting_g01_c01.avi, v_Rafting_g01_c02.avi, ...) RockClimbingIndoor 144 videos (v_RockClimbingIndoor_g01_c01.avi, v_RockClimbingIndoor_g01_c02.avi, ...) RopeClimbing 119 videos (v_RopeClimbing_g01_c01.avi, v_RopeClimbing_g01_c02.avi, ...) Rowing 137 videos (v_Rowing_g01_c01.avi, v_Rowing_g01_c02.avi, ...) SalsaSpin 133 videos (v_SalsaSpin_g01_c01.avi, v_SalsaSpin_g01_c02.avi, ...) ShavingBeard 161 videos (v_ShavingBeard_g01_c01.avi, v_ShavingBeard_g01_c02.avi, ...) Shotput 144 videos (v_Shotput_g01_c01.avi, v_Shotput_g01_c02.avi, ...) SkateBoarding 120 videos (v_SkateBoarding_g01_c01.avi, v_SkateBoarding_g01_c02.avi, ...) Skiing 135 videos (v_Skiing_g01_c01.avi, v_Skiing_g01_c02.avi, ...) Skijet 100 videos (v_Skijet_g01_c01.avi, v_Skijet_g01_c02.avi, ...) SkyDiving 110 videos (v_SkyDiving_g01_c01.avi, v_SkyDiving_g01_c02.avi, ...) SoccerJuggling 147 videos (v_SoccerJuggling_g01_c01.avi, v_SoccerJuggling_g01_c02.avi, ...) SoccerPenalty 137 videos (v_SoccerPenalty_g01_c01.avi, v_SoccerPenalty_g01_c02.avi, ...) StillRings 112 videos (v_StillRings_g01_c01.avi, v_StillRings_g01_c02.avi, ...) SumoWrestling 116 videos (v_SumoWrestling_g01_c01.avi, v_SumoWrestling_g01_c02.avi, ...) Surfing 126 videos (v_Surfing_g01_c01.avi, v_Surfing_g01_c02.avi, ...) Swing 131 videos (v_Swing_g01_c01.avi, v_Swing_g01_c02.avi, ...) TableTennisShot 140 videos (v_TableTennisShot_g01_c01.avi, v_TableTennisShot_g01_c02.avi, ...) TaiChi 100 videos (v_TaiChi_g01_c01.avi, v_TaiChi_g01_c02.avi, ...) TennisSwing 166 videos (v_TennisSwing_g01_c01.avi, v_TennisSwing_g01_c02.avi, ...) ThrowDiscus 130 videos (v_ThrowDiscus_g01_c01.avi, v_ThrowDiscus_g01_c02.avi, ...) TrampolineJumping 119 videos (v_TrampolineJumping_g01_c01.avi, v_TrampolineJumping_g01_c02.avi, ...) Typing 136 videos (v_Typing_g01_c01.avi, v_Typing_g01_c02.avi, ...) UnevenBars 104 videos (v_UnevenBars_g01_c01.avi, v_UnevenBars_g01_c02.avi, ...) VolleyballSpiking 116 videos (v_VolleyballSpiking_g01_c01.avi, v_VolleyballSpiking_g01_c02.avi, ...) WalkingWithDog 123 videos (v_WalkingWithDog_g01_c01.avi, v_WalkingWithDog_g01_c02.avi, ...) WallPushups 130 videos (v_WallPushups_g01_c01.avi, v_WallPushups_g01_c02.avi, ...) WritingOnBoard 152 videos (v_WritingOnBoard_g01_c01.avi, v_WritingOnBoard_g01_c02.avi, ...) YoYo 128 videos (v_YoYo_g01_c01.avi, v_YoYo_g01_c02.avi, ...)
# Get a sample cricket video.
video_path = fetch_ucf_video("v_CricketShot_g04_c02.avi")
sample_video = load_video(video_path)
Fetching https://www.crcv.ucf.edu/THUMOS14/UCF101/UCF101/v_CricketShot_g04_c02.avi => /tmpfs/tmp/tmpc624wg54/v_CricketShot_g04_c02.avi
sample_video.shape
(116, 224, 224, 3)
i3d = hub.load("https://tfhub.dev/deepmind/i3d-kinetics-400/1").signatures['default']
id3 모델을 실행하고 상위 5개의 행동 예측값을 출력합니다.
def predict(sample_video):
# Add a batch axis to the sample video.
model_input = tf.constant(sample_video, dtype=tf.float32)[tf.newaxis, ...]
logits = i3d(model_input)['default'][0]
probabilities = tf.nn.softmax(logits)
print("Top 5 actions:")
for i in np.argsort(probabilities)[::-1][:5]:
print(f" {labels[i]:22}: {probabilities[i] * 100:5.2f}%")
predict(sample_video)
Top 5 actions: playing cricket : 97.77% skateboarding : 0.71% robot dancing : 0.56% roller skating : 0.56% golf putting : 0.13%
이제 https://commons.wikimedia.org/wiki/Category:Videos_of_sports에서 새 비디오를 시도해 봅니다.
Patrick Gillett의 이 비디오는 어떻습니까?
curl -O https://upload.wikimedia.org/wikipedia/commons/8/86/End_of_a_jam.ogv
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 55.0M 100 55.0M 0 0 25.9M 0 0:00:02 0:00:02 --:--:-- 25.9M
video_path = "End_of_a_jam.ogv"
sample_video = load_video(video_path)[:100]
sample_video.shape
(100, 224, 224, 3)
to_gif(sample_video)
predict(sample_video)
Top 5 actions: roller skating : 96.85% playing volleyball : 1.63% skateboarding : 0.21% playing ice hockey : 0.20% playing basketball : 0.16%