Reconocimiento de acciones con una CNN 3D inflada

Ver en TensorFlow.org Ejecutar en Google Colab Ver en GitHub Descargar cuaderno Ver modelo TF Hub

Esto demuestra Colab acciones que reconocen en los datos de vídeo utilizando el tfhub.dev/deepmind/i3d-kinetics-400/1 módulo. Más modelos para detectar acciones en los vídeos se pueden encontrar aquí .

El modelo subyacente se describe en el documento " Quo Vadis, Reconocimiento de acciones? Un nuevo modelo y la cinética del conjunto de datos " Joao Carreira y Andrew Zisserman. El documento se publicó en arXiv en mayo de 2017 y se publicó como un documento de conferencia CVPR 2017. El código fuente está disponible públicamente en GitHub .

"Quo Vadis" introdujo una nueva arquitectura para la clasificación de video, Inflated 3D Convnet o I3D. Esta arquitectura logró resultados de vanguardia en los conjuntos de datos UCF101 y HMDB51 al ajustar estos modelos. I3D modelos pre-formados sobre la cinética también colocó por primera vez en el 2017 CVPR desafío Charades .

El módulo original fue entrenado en la dateset cinética-400 y sabe de 400 acciones diferentes. Las etiquetas para estas acciones se pueden encontrar en el archivo de mapa de etiquetas .

En este Colab lo usaremos para reconocer actividades en videos de un conjunto de datos UCF101.

Configuración

pip install -q imageio
pip install -q opencv-python
pip install -q git+https://github.com/tensorflow/docs

Importar los módulos necesarios

# TensorFlow and TF-Hub modules.
from absl import logging

import tensorflow as tf
import tensorflow_hub as hub
from tensorflow_docs.vis import embed

logging.set_verbosity(logging.ERROR)

# Some modules to help with reading the UCF101 dataset.
import random
import re
import os
import tempfile
import ssl
import cv2
import numpy as np

# Some modules to display an animation using imageio.
import imageio
from IPython import display

from urllib import request  # requires python3

Funciones auxiliares para el conjunto de datos UCF101

# Utilities to fetch videos from UCF101 dataset
UCF_ROOT = "https://www.crcv.ucf.edu/THUMOS14/UCF101/UCF101/"
_VIDEO_LIST = None
_CACHE_DIR = tempfile.mkdtemp()
# As of July 2020, crcv.ucf.edu doesn't use a certificate accepted by the
# default Colab environment anymore.
unverified_context = ssl._create_unverified_context()

def list_ucf_videos():
  """Lists videos available in UCF101 dataset."""
  global _VIDEO_LIST
  if not _VIDEO_LIST:
    index = request.urlopen(UCF_ROOT, context=unverified_context).read().decode("utf-8")
    videos = re.findall("(v_[\w_]+\.avi)", index)
    _VIDEO_LIST = sorted(set(videos))
  return list(_VIDEO_LIST)

def fetch_ucf_video(video):
  """Fetchs a video and cache into local filesystem."""
  cache_path = os.path.join(_CACHE_DIR, video)
  if not os.path.exists(cache_path):
    urlpath = request.urljoin(UCF_ROOT, video)
    print("Fetching %s => %s" % (urlpath, cache_path))
    data = request.urlopen(urlpath, context=unverified_context).read()
    open(cache_path, "wb").write(data)
  return cache_path

# Utilities to open video files using CV2
def crop_center_square(frame):
  y, x = frame.shape[0:2]
  min_dim = min(y, x)
  start_x = (x // 2) - (min_dim // 2)
  start_y = (y // 2) - (min_dim // 2)
  return frame[start_y:start_y+min_dim,start_x:start_x+min_dim]

def load_video(path, max_frames=0, resize=(224, 224)):
  cap = cv2.VideoCapture(path)
  frames = []
  try:
    while True:
      ret, frame = cap.read()
      if not ret:
        break
      frame = crop_center_square(frame)
      frame = cv2.resize(frame, resize)
      frame = frame[:, :, [2, 1, 0]]
      frames.append(frame)

      if len(frames) == max_frames:
        break
  finally:
    cap.release()
  return np.array(frames) / 255.0

def to_gif(images):
  converted_images = np.clip(images * 255, 0, 255).astype(np.uint8)
  imageio.mimsave('./animation.gif', converted_images, fps=25)
  return embed.embed_file('./animation.gif')

Obtenga las etiquetas kinetics-400

Found 400 labels.

Usando el conjunto de datos UCF101

# Get the list of videos in the dataset.
ucf_videos = list_ucf_videos()

categories = {}
for video in ucf_videos:
  category = video[2:-12]
  if category not in categories:
    categories[category] = []
  categories[category].append(video)
print("Found %d videos in %d categories." % (len(ucf_videos), len(categories)))

for category, sequences in categories.items():
  summary = ", ".join(sequences[:2])
  print("%-20s %4d videos (%s, ...)" % (category, len(sequences), summary))
Found 13320 videos in 101 categories.
ApplyEyeMakeup        145 videos (v_ApplyEyeMakeup_g01_c01.avi, v_ApplyEyeMakeup_g01_c02.avi, ...)
ApplyLipstick         114 videos (v_ApplyLipstick_g01_c01.avi, v_ApplyLipstick_g01_c02.avi, ...)
Archery               145 videos (v_Archery_g01_c01.avi, v_Archery_g01_c02.avi, ...)
BabyCrawling          132 videos (v_BabyCrawling_g01_c01.avi, v_BabyCrawling_g01_c02.avi, ...)
BalanceBeam           108 videos (v_BalanceBeam_g01_c01.avi, v_BalanceBeam_g01_c02.avi, ...)
BandMarching          155 videos (v_BandMarching_g01_c01.avi, v_BandMarching_g01_c02.avi, ...)
BaseballPitch         150 videos (v_BaseballPitch_g01_c01.avi, v_BaseballPitch_g01_c02.avi, ...)
BasketballDunk        131 videos (v_BasketballDunk_g01_c01.avi, v_BasketballDunk_g01_c02.avi, ...)
Basketball            134 videos (v_Basketball_g01_c01.avi, v_Basketball_g01_c02.avi, ...)
BenchPress            160 videos (v_BenchPress_g01_c01.avi, v_BenchPress_g01_c02.avi, ...)
Biking                134 videos (v_Biking_g01_c01.avi, v_Biking_g01_c02.avi, ...)
Billiards             150 videos (v_Billiards_g01_c01.avi, v_Billiards_g01_c02.avi, ...)
BlowDryHair           131 videos (v_BlowDryHair_g01_c01.avi, v_BlowDryHair_g01_c02.avi, ...)
BlowingCandles        109 videos (v_BlowingCandles_g01_c01.avi, v_BlowingCandles_g01_c02.avi, ...)
BodyWeightSquats      112 videos (v_BodyWeightSquats_g01_c01.avi, v_BodyWeightSquats_g01_c02.avi, ...)
Bowling               155 videos (v_Bowling_g01_c01.avi, v_Bowling_g01_c02.avi, ...)
BoxingPunchingBag     163 videos (v_BoxingPunchingBag_g01_c01.avi, v_BoxingPunchingBag_g01_c02.avi, ...)
BoxingSpeedBag        134 videos (v_BoxingSpeedBag_g01_c01.avi, v_BoxingSpeedBag_g01_c02.avi, ...)
BreastStroke          101 videos (v_BreastStroke_g01_c01.avi, v_BreastStroke_g01_c02.avi, ...)
BrushingTeeth         131 videos (v_BrushingTeeth_g01_c01.avi, v_BrushingTeeth_g01_c02.avi, ...)
CleanAndJerk          112 videos (v_CleanAndJerk_g01_c01.avi, v_CleanAndJerk_g01_c02.avi, ...)
CliffDiving           138 videos (v_CliffDiving_g01_c01.avi, v_CliffDiving_g01_c02.avi, ...)
CricketBowling        139 videos (v_CricketBowling_g01_c01.avi, v_CricketBowling_g01_c02.avi, ...)
CricketShot           167 videos (v_CricketShot_g01_c01.avi, v_CricketShot_g01_c02.avi, ...)
CuttingInKitchen      110 videos (v_CuttingInKitchen_g01_c01.avi, v_CuttingInKitchen_g01_c02.avi, ...)
Diving                150 videos (v_Diving_g01_c01.avi, v_Diving_g01_c02.avi, ...)
Drumming              161 videos (v_Drumming_g01_c01.avi, v_Drumming_g01_c02.avi, ...)
Fencing               111 videos (v_Fencing_g01_c01.avi, v_Fencing_g01_c02.avi, ...)
FieldHockeyPenalty    126 videos (v_FieldHockeyPenalty_g01_c01.avi, v_FieldHockeyPenalty_g01_c02.avi, ...)
FloorGymnastics       125 videos (v_FloorGymnastics_g01_c01.avi, v_FloorGymnastics_g01_c02.avi, ...)
FrisbeeCatch          126 videos (v_FrisbeeCatch_g01_c01.avi, v_FrisbeeCatch_g01_c02.avi, ...)
FrontCrawl            137 videos (v_FrontCrawl_g01_c01.avi, v_FrontCrawl_g01_c02.avi, ...)
GolfSwing             139 videos (v_GolfSwing_g01_c01.avi, v_GolfSwing_g01_c02.avi, ...)
Haircut               130 videos (v_Haircut_g01_c01.avi, v_Haircut_g01_c02.avi, ...)
HammerThrow           150 videos (v_HammerThrow_g01_c01.avi, v_HammerThrow_g01_c02.avi, ...)
Hammering             140 videos (v_Hammering_g01_c01.avi, v_Hammering_g01_c02.avi, ...)
HandstandPushups      128 videos (v_HandstandPushups_g01_c01.avi, v_HandstandPushups_g01_c02.avi, ...)
HandstandWalking      111 videos (v_HandstandWalking_g01_c01.avi, v_HandstandWalking_g01_c02.avi, ...)
HeadMassage           147 videos (v_HeadMassage_g01_c01.avi, v_HeadMassage_g01_c02.avi, ...)
HighJump              123 videos (v_HighJump_g01_c01.avi, v_HighJump_g01_c02.avi, ...)
HorseRace             124 videos (v_HorseRace_g01_c01.avi, v_HorseRace_g01_c02.avi, ...)
HorseRiding           164 videos (v_HorseRiding_g01_c01.avi, v_HorseRiding_g01_c02.avi, ...)
HulaHoop              125 videos (v_HulaHoop_g01_c01.avi, v_HulaHoop_g01_c02.avi, ...)
IceDancing            158 videos (v_IceDancing_g01_c01.avi, v_IceDancing_g01_c02.avi, ...)
JavelinThrow          117 videos (v_JavelinThrow_g01_c01.avi, v_JavelinThrow_g01_c02.avi, ...)
JugglingBalls         121 videos (v_JugglingBalls_g01_c01.avi, v_JugglingBalls_g01_c02.avi, ...)
JumpRope              144 videos (v_JumpRope_g01_c01.avi, v_JumpRope_g01_c02.avi, ...)
JumpingJack           123 videos (v_JumpingJack_g01_c01.avi, v_JumpingJack_g01_c02.avi, ...)
Kayaking              141 videos (v_Kayaking_g01_c01.avi, v_Kayaking_g01_c02.avi, ...)
Knitting              123 videos (v_Knitting_g01_c01.avi, v_Knitting_g01_c02.avi, ...)
LongJump              131 videos (v_LongJump_g01_c01.avi, v_LongJump_g01_c02.avi, ...)
Lunges                127 videos (v_Lunges_g01_c01.avi, v_Lunges_g01_c02.avi, ...)
MilitaryParade        125 videos (v_MilitaryParade_g01_c01.avi, v_MilitaryParade_g01_c02.avi, ...)
Mixing                136 videos (v_Mixing_g01_c01.avi, v_Mixing_g01_c02.avi, ...)
MoppingFloor          110 videos (v_MoppingFloor_g01_c01.avi, v_MoppingFloor_g01_c02.avi, ...)
Nunchucks             132 videos (v_Nunchucks_g01_c01.avi, v_Nunchucks_g01_c02.avi, ...)
ParallelBars          114 videos (v_ParallelBars_g01_c01.avi, v_ParallelBars_g01_c02.avi, ...)
PizzaTossing          113 videos (v_PizzaTossing_g01_c01.avi, v_PizzaTossing_g01_c02.avi, ...)
PlayingCello          164 videos (v_PlayingCello_g01_c01.avi, v_PlayingCello_g01_c02.avi, ...)
PlayingDaf            151 videos (v_PlayingDaf_g01_c01.avi, v_PlayingDaf_g01_c02.avi, ...)
PlayingDhol           164 videos (v_PlayingDhol_g01_c01.avi, v_PlayingDhol_g01_c02.avi, ...)
PlayingFlute          155 videos (v_PlayingFlute_g01_c01.avi, v_PlayingFlute_g01_c02.avi, ...)
PlayingGuitar         160 videos (v_PlayingGuitar_g01_c01.avi, v_PlayingGuitar_g01_c02.avi, ...)
PlayingPiano          105 videos (v_PlayingPiano_g01_c01.avi, v_PlayingPiano_g01_c02.avi, ...)
PlayingSitar          157 videos (v_PlayingSitar_g01_c01.avi, v_PlayingSitar_g01_c02.avi, ...)
PlayingTabla          111 videos (v_PlayingTabla_g01_c01.avi, v_PlayingTabla_g01_c02.avi, ...)
PlayingViolin         100 videos (v_PlayingViolin_g01_c01.avi, v_PlayingViolin_g01_c02.avi, ...)
PoleVault             149 videos (v_PoleVault_g01_c01.avi, v_PoleVault_g01_c02.avi, ...)
PommelHorse           123 videos (v_PommelHorse_g01_c01.avi, v_PommelHorse_g01_c02.avi, ...)
PullUps               100 videos (v_PullUps_g01_c01.avi, v_PullUps_g01_c02.avi, ...)
Punch                 160 videos (v_Punch_g01_c01.avi, v_Punch_g01_c02.avi, ...)
PushUps               102 videos (v_PushUps_g01_c01.avi, v_PushUps_g01_c02.avi, ...)
Rafting               111 videos (v_Rafting_g01_c01.avi, v_Rafting_g01_c02.avi, ...)
RockClimbingIndoor    144 videos (v_RockClimbingIndoor_g01_c01.avi, v_RockClimbingIndoor_g01_c02.avi, ...)
RopeClimbing          119 videos (v_RopeClimbing_g01_c01.avi, v_RopeClimbing_g01_c02.avi, ...)
Rowing                137 videos (v_Rowing_g01_c01.avi, v_Rowing_g01_c02.avi, ...)
SalsaSpin             133 videos (v_SalsaSpin_g01_c01.avi, v_SalsaSpin_g01_c02.avi, ...)
ShavingBeard          161 videos (v_ShavingBeard_g01_c01.avi, v_ShavingBeard_g01_c02.avi, ...)
Shotput               144 videos (v_Shotput_g01_c01.avi, v_Shotput_g01_c02.avi, ...)
SkateBoarding         120 videos (v_SkateBoarding_g01_c01.avi, v_SkateBoarding_g01_c02.avi, ...)
Skiing                135 videos (v_Skiing_g01_c01.avi, v_Skiing_g01_c02.avi, ...)
Skijet                100 videos (v_Skijet_g01_c01.avi, v_Skijet_g01_c02.avi, ...)
SkyDiving             110 videos (v_SkyDiving_g01_c01.avi, v_SkyDiving_g01_c02.avi, ...)
SoccerJuggling        147 videos (v_SoccerJuggling_g01_c01.avi, v_SoccerJuggling_g01_c02.avi, ...)
SoccerPenalty         137 videos (v_SoccerPenalty_g01_c01.avi, v_SoccerPenalty_g01_c02.avi, ...)
StillRings            112 videos (v_StillRings_g01_c01.avi, v_StillRings_g01_c02.avi, ...)
SumoWrestling         116 videos (v_SumoWrestling_g01_c01.avi, v_SumoWrestling_g01_c02.avi, ...)
Surfing               126 videos (v_Surfing_g01_c01.avi, v_Surfing_g01_c02.avi, ...)
Swing                 131 videos (v_Swing_g01_c01.avi, v_Swing_g01_c02.avi, ...)
TableTennisShot       140 videos (v_TableTennisShot_g01_c01.avi, v_TableTennisShot_g01_c02.avi, ...)
TaiChi                100 videos (v_TaiChi_g01_c01.avi, v_TaiChi_g01_c02.avi, ...)
TennisSwing           166 videos (v_TennisSwing_g01_c01.avi, v_TennisSwing_g01_c02.avi, ...)
ThrowDiscus           130 videos (v_ThrowDiscus_g01_c01.avi, v_ThrowDiscus_g01_c02.avi, ...)
TrampolineJumping     119 videos (v_TrampolineJumping_g01_c01.avi, v_TrampolineJumping_g01_c02.avi, ...)
Typing                136 videos (v_Typing_g01_c01.avi, v_Typing_g01_c02.avi, ...)
UnevenBars            104 videos (v_UnevenBars_g01_c01.avi, v_UnevenBars_g01_c02.avi, ...)
VolleyballSpiking     116 videos (v_VolleyballSpiking_g01_c01.avi, v_VolleyballSpiking_g01_c02.avi, ...)
WalkingWithDog        123 videos (v_WalkingWithDog_g01_c01.avi, v_WalkingWithDog_g01_c02.avi, ...)
WallPushups           130 videos (v_WallPushups_g01_c01.avi, v_WallPushups_g01_c02.avi, ...)
WritingOnBoard        152 videos (v_WritingOnBoard_g01_c01.avi, v_WritingOnBoard_g01_c02.avi, ...)
YoYo                  128 videos (v_YoYo_g01_c01.avi, v_YoYo_g01_c02.avi, ...)
# Get a sample cricket video.
video_path = fetch_ucf_video("v_CricketShot_g04_c02.avi")
sample_video = load_video(video_path)
Fetching https://www.crcv.ucf.edu/THUMOS14/UCF101/UCF101/v_CricketShot_g04_c02.avi => /tmp/tmp4wns8m2b/v_CricketShot_g04_c02.avi
sample_video.shape
(116, 224, 224, 3)
i3d = hub.load("https://tfhub.dev/deepmind/i3d-kinetics-400/1").signatures['default']

Ejecute el modelo id3 e imprima las 5 predicciones de acción principales.

def predict(sample_video):
  # Add a batch axis to the sample video.
  model_input = tf.constant(sample_video, dtype=tf.float32)[tf.newaxis, ...]

  logits = i3d(model_input)['default'][0]
  probabilities = tf.nn.softmax(logits)

  print("Top 5 actions:")
  for i in np.argsort(probabilities)[::-1][:5]:
    print(f"  {labels[i]:22}: {probabilities[i] * 100:5.2f}%")
predict(sample_video)
Top 5 actions:
  playing cricket       : 97.77%
  skateboarding         :  0.71%
  robot dancing         :  0.56%
  roller skating        :  0.56%
  golf putting          :  0.13%

Ahora trata de un nuevo video, a partir de: https://commons.wikimedia.org/wiki/Category : Videos_of_sports

¿Qué hay de este video por Patrick Gillett:

curl -O https://upload.wikimedia.org/wikipedia/commons/8/86/End_of_a_jam.ogv
% Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 55.0M  100 55.0M    0     0  21.9M      0  0:00:02  0:00:02 --:--:-- 21.9M
video_path = "End_of_a_jam.ogv"
sample_video = load_video(video_path)[:100]
sample_video.shape
(100, 224, 224, 3)
to_gif(sample_video)

gif

predict(sample_video)
Top 5 actions:
  roller skating        : 96.85%
  playing volleyball    :  1.63%
  skateboarding         :  0.21%
  playing ice hockey    :  0.20%
  playing basketball    :  0.16%