Video_Background_Substitution.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Video_Background_Substitution.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMKcDVrWrOhPGku7+NiU0cg",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/HaithemH/videos_background_substitution/blob/master/Video_Background_Substitution.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "arSJs-jT6kQW"
      },
      "source": [
        "# Run with Colab"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JGB00GKc6k2-",
        "outputId": "c1f4d439-dab7-4863-bfc3-41012f58106e"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2PaUfUxDsFVr",
        "outputId": "21af6ee2-9815-44dc-8020-9f306261dad1"
      },
      "source": [
        "#clone the repo \n",
        "!git clone https://github.com/HaithemH/videos_background_substitution\n",
        "%cd videos_background_substitution\n",
        "!ls -a         "
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'videos_background_substitution'...\n",
            "remote: Enumerating objects: 74, done.\u001b[K\n",
            "remote: Counting objects: 100% (74/74), done.\u001b[K\n",
            "remote: Compressing objects: 100% (72/72), done.\u001b[K\n",
            "remote: Total 74 (delta 30), reused 27 (delta 1), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (74/74), done.\n",
            "/content/videos_background_substitution\n",
            ".   _assets\tDockerfile  inputs   models   README.md\t\tscripts\n",
            "..  background\t.git\t    LICENSE  outputs  requirements.txt\ttest\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s4jzDZsLtNWr",
        "outputId": "57102197-55df-4145-eb2b-e0f852782d82"
      },
      "source": [
        "#Install dependencies\n",
        "!pip install -r requirements.txt"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 1)) (1.19.5)\n",
            "Collecting configparser==4.0.2\n",
            "  Downloading configparser-4.0.2-py2.py3-none-any.whl (22 kB)\n",
            "Requirement already satisfied: opencv-python in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 3)) (4.1.2.30)\n",
            "Collecting pixellib\n",
            "  Downloading pixellib-0.6.6-py3-none-any.whl (69 kB)\n",
            "\u001b[K     |████████████████████████████████| 69 kB 3.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: sklearn in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 5)) (0.0)\n",
            "Requirement already satisfied: imgaug in /usr/local/lib/python3.7/dist-packages (from pixellib->-r requirements.txt (line 4)) (0.2.9)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from pixellib->-r requirements.txt (line 4)) (3.2.2)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.7/dist-packages (from pixellib->-r requirements.txt (line 4)) (7.1.2)\n",
            "Collecting labelme2coco\n",
            "  Downloading labelme2coco-0.1.2-py3-none-any.whl (19 kB)\n",
            "Collecting imantics\n",
            "  Downloading imantics-0.1.12.tar.gz (13 kB)\n",
            "Requirement already satisfied: scikit-image in /usr/local/lib/python3.7/dist-packages (from pixellib->-r requirements.txt (line 4)) (0.16.2)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from sklearn->-r requirements.txt (line 5)) (0.22.2.post1)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.7/dist-packages (from imantics->pixellib->-r requirements.txt (line 4)) (4.2.6)\n",
            "Collecting xmljson\n",
            "  Downloading xmljson-0.2.1-py2.py3-none-any.whl (10 kB)\n",
            "Requirement already satisfied: imageio in /usr/local/lib/python3.7/dist-packages (from imgaug->pixellib->-r requirements.txt (line 4)) (2.4.1)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from imgaug->pixellib->-r requirements.txt (line 4)) (1.4.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from imgaug->pixellib->-r requirements.txt (line 4)) (1.15.0)\n",
            "Requirement already satisfied: Shapely in /usr/local/lib/python3.7/dist-packages (from imgaug->pixellib->-r requirements.txt (line 4)) (1.7.1)\n",
            "Requirement already satisfied: PyWavelets>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from scikit-image->pixellib->-r requirements.txt (line 4)) (1.1.1)\n",
            "Requirement already satisfied: networkx>=2.0 in /usr/local/lib/python3.7/dist-packages (from scikit-image->pixellib->-r requirements.txt (line 4)) (2.6.3)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->pixellib->-r requirements.txt (line 4)) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->pixellib->-r requirements.txt (line 4)) (1.3.2)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->pixellib->-r requirements.txt (line 4)) (2.4.7)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->pixellib->-r requirements.txt (line 4)) (2.8.2)\n",
            "Requirement already satisfied: jsonschema>=2.6.0 in /usr/local/lib/python3.7/dist-packages (from labelme2coco->pixellib->-r requirements.txt (line 4)) (2.6.0)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->sklearn->-r requirements.txt (line 5)) (1.0.1)\n",
            "Building wheels for collected packages: imantics\n",
            "  Building wheel for imantics (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for imantics: filename=imantics-0.1.12-py3-none-any.whl size=16031 sha256=48502b6d2fe935d01622ec8a7339433fa1521ba2ab2279ab6ad3b27475319cc7\n",
            "  Stored in directory: /root/.cache/pip/wheels/da/7c/3e/296fe3ed4eb3bd713e91dee0d0549f12f316d49939a64bdc96\n",
            "Successfully built imantics\n",
            "Installing collected packages: xmljson, labelme2coco, imantics, pixellib, configparser\n",
            "Successfully installed configparser-4.0.2 imantics-0.1.12 labelme2coco-0.1.2 pixellib-0.6.6 xmljson-0.2.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C6y1NiLz0uf6"
      },
      "source": [
        "#Run background extraction\n",
        "#Note: modify vodeo paths in config.json, the algorithm \"bg_algo\" can be median or gmm\n",
        "!python ./scripts/bgsubtractor.py"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uDyTT6BF3set"
      },
      "source": [
        "#Run moving objects extraction\n",
        "#Note: modify vodeo paths in config.json, the algorithm \"mo_fg_algo\" can be mog or knn\n",
        "!python ./scripts/mo_subtractor.py"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1mWYtLnl4PGT"
      },
      "source": [
        "#Replace background with GrauCat\n",
        "#Note: modify vodeo paths in config.json, grubcut_iter:\"number of iterations\" \"grubcut_ROI\":region of interest\n",
        "!python ./scripts/grubcut_combiner.py"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S3Qr-LgMRJD-"
      },
      "source": [
        "#Replace background with DeeplabPascalvoc\n",
        "#download the model\n",
        "import requests \n",
        "file_url = \"https://github.com/ayoolaolafenwa/PixelLib/releases/download/1.1/xception_pascalvoc.pb\"\n",
        "\t\n",
        "r = requests.get(file_url, stream = True) \n",
        "\n",
        "with open(\"/content/videos_background_substitution/models/xception_pascalvoc.pb\", \"wb\") as file: \n",
        "\tfor block in r.iter_content(chunk_size = 1024): \n",
        "\t\tif block: \n",
        "\t\t\tfile.write(block) \n",
        "#Run inference for background replacement   \n",
        "!python ./scripts/pascal_voc_combiner.py"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sxlM1Ucb6b0U"
      },
      "source": [
        "# Docker Container "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j58379LHErLe",
        "outputId": "8fd3d4ba-2110-44b2-8aac-4071f15211de"
      },
      "source": [
        "# Install Docker.\n",
        "!apt-get -qq install docker.io"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Preconfiguring packages ...\n",
            "Selecting previously unselected package libnfnetlink0:amd64.\n",
            "(Reading database ... 155013 files and directories currently installed.)\n",
            "Preparing to unpack .../00-libnfnetlink0_1.0.1-3_amd64.deb ...\n",
            "Unpacking libnfnetlink0:amd64 (1.0.1-3) ...\n",
            "Selecting previously unselected package pigz.\n",
            "Preparing to unpack .../01-pigz_2.4-1_amd64.deb ...\n",
            "Unpacking pigz (2.4-1) ...\n",
            "Selecting previously unselected package libmnl0:amd64.\n",
            "Preparing to unpack .../02-libmnl0_1.0.4-2_amd64.deb ...\n",
            "Unpacking libmnl0:amd64 (1.0.4-2) ...\n",
            "Selecting previously unselected package iproute2.\n",
            "Preparing to unpack .../03-iproute2_4.15.0-2ubuntu1.3_amd64.deb ...\n",
            "Unpacking iproute2 (4.15.0-2ubuntu1.3) ...\n",
            "Selecting previously unselected package libatm1:amd64.\n",
            "Preparing to unpack .../04-libatm1_1%3a2.5.1-2build1_amd64.deb ...\n",
            "Unpacking libatm1:amd64 (1:2.5.1-2build1) ...\n",
            "Selecting previously unselected package libxtables12:amd64.\n",
            "Preparing to unpack .../05-libxtables12_1.6.1-2ubuntu2_amd64.deb ...\n",
            "Unpacking libxtables12:amd64 (1.6.1-2ubuntu2) ...\n",
            "Selecting previously unselected package apparmor.\n",
            "Preparing to unpack .../06-apparmor_2.12-4ubuntu5.1_amd64.deb ...\n",
            "Unpacking apparmor (2.12-4ubuntu5.1) ...\n",
            "Selecting previously unselected package libip6tc0:amd64.\n",
            "Preparing to unpack .../07-libip6tc0_1.6.1-2ubuntu2_amd64.deb ...\n",
            "Unpacking libip6tc0:amd64 (1.6.1-2ubuntu2) ...\n",
            "Selecting previously unselected package libiptc0:amd64.\n",
            "Preparing to unpack .../08-libiptc0_1.6.1-2ubuntu2_amd64.deb ...\n",
            "Unpacking libiptc0:amd64 (1.6.1-2ubuntu2) ...\n",
            "Selecting previously unselected package libnetfilter-conntrack3:amd64.\n",
            "Preparing to unpack .../09-libnetfilter-conntrack3_1.0.6-2_amd64.deb ...\n",
            "Unpacking libnetfilter-conntrack3:amd64 (1.0.6-2) ...\n",
            "Selecting previously unselected package iptables.\n",
            "Preparing to unpack .../10-iptables_1.6.1-2ubuntu2_amd64.deb ...\n",
            "Unpacking iptables (1.6.1-2ubuntu2) ...\n",
            "Selecting previously unselected package bridge-utils.\n",
            "Preparing to unpack .../11-bridge-utils_1.5-15ubuntu1_amd64.deb ...\n",
            "Unpacking bridge-utils (1.5-15ubuntu1) ...\n",
            "Selecting previously unselected package runc.\n",
            "Preparing to unpack .../12-runc_1.0.0~rc95-0ubuntu1~18.04.2_amd64.deb ...\n",
            "Unpacking runc (1.0.0~rc95-0ubuntu1~18.04.2) ...\n",
            "Selecting previously unselected package containerd.\n",
            "Preparing to unpack .../13-containerd_1.5.2-0ubuntu1~18.04.2_amd64.deb ...\n",
            "Unpacking containerd (1.5.2-0ubuntu1~18.04.2) ...\n",
            "Selecting previously unselected package dns-root-data.\n",
            "Preparing to unpack .../14-dns-root-data_2018013001_all.deb ...\n",
            "Unpacking dns-root-data (2018013001) ...\n",
            "Selecting previously unselected package dnsmasq-base.\n",
            "Preparing to unpack .../15-dnsmasq-base_2.79-1ubuntu0.4_amd64.deb ...\n",
            "Unpacking dnsmasq-base (2.79-1ubuntu0.4) ...\n",
            "Selecting previously unselected package docker.io.\n",
            "Preparing to unpack .../16-docker.io_20.10.7-0ubuntu1~18.04.1_amd64.deb ...\n",
            "Unpacking docker.io (20.10.7-0ubuntu1~18.04.1) ...\n",
            "Selecting previously unselected package netcat-traditional.\n",
            "Preparing to unpack .../17-netcat-traditional_1.10-41.1_amd64.deb ...\n",
            "Unpacking netcat-traditional (1.10-41.1) ...\n",
            "Selecting previously unselected package netcat.\n",
            "Preparing to unpack .../18-netcat_1.10-41.1_all.deb ...\n",
            "Unpacking netcat (1.10-41.1) ...\n",
            "Selecting previously unselected package ubuntu-fan.\n",
            "Preparing to unpack .../19-ubuntu-fan_0.12.10_all.deb ...\n",
            "Unpacking ubuntu-fan (0.12.10) ...\n",
            "Setting up runc (1.0.0~rc95-0ubuntu1~18.04.2) ...\n",
            "Setting up dns-root-data (2018013001) ...\n",
            "Setting up netcat-traditional (1.10-41.1) ...\n",
            "update-alternatives: using /bin/nc.traditional to provide /bin/nc (nc) in auto mode\n",
            "Setting up libiptc0:amd64 (1.6.1-2ubuntu2) ...\n",
            "Setting up apparmor (2.12-4ubuntu5.1) ...\n",
            "Created symlink /etc/systemd/system/sysinit.target.wants/apparmor.service → /lib/systemd/system/apparmor.service.\n",
            "Setting up containerd (1.5.2-0ubuntu1~18.04.2) ...\n",
            "Created symlink /etc/systemd/system/multi-user.target.wants/containerd.service → /lib/systemd/system/containerd.service.\n",
            "Setting up bridge-utils (1.5-15ubuntu1) ...\n",
            "Setting up libatm1:amd64 (1:2.5.1-2build1) ...\n",
            "Setting up libxtables12:amd64 (1.6.1-2ubuntu2) ...\n",
            "Setting up netcat (1.10-41.1) ...\n",
            "Setting up libnfnetlink0:amd64 (1.0.1-3) ...\n",
            "Setting up libmnl0:amd64 (1.0.4-2) ...\n",
            "Setting up pigz (2.4-1) ...\n",
            "Setting up libip6tc0:amd64 (1.6.1-2ubuntu2) ...\n",
            "Setting up libnetfilter-conntrack3:amd64 (1.0.6-2) ...\n",
            "Setting up iproute2 (4.15.0-2ubuntu1.3) ...\n",
            "Setting up iptables (1.6.1-2ubuntu2) ...\n",
            "Setting up dnsmasq-base (2.79-1ubuntu0.4) ...\n",
            "Setting up docker.io (20.10.7-0ubuntu1~18.04.1) ...\n",
            "Adding group `docker' (GID 109) ...\n",
            "Done.\n",
            "Created symlink /etc/systemd/system/multi-user.target.wants/docker.service → /lib/systemd/system/docker.service.\n",
            "Created symlink /etc/systemd/system/sockets.target.wants/docker.socket → /lib/systemd/system/docker.socket.\n",
            "invoke-rc.d: unknown initscript, /etc/init.d/docker not found.\n",
            "invoke-rc.d: could not determine current runlevel\n",
            "Setting up ubuntu-fan (0.12.10) ...\n",
            "Created symlink /etc/systemd/system/multi-user.target.wants/ubuntu-fan.service → /lib/systemd/system/ubuntu-fan.service.\n",
            "invoke-rc.d: could not determine current runlevel\n",
            "invoke-rc.d: policy-rc.d denied execution of start.\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Processing triggers for dbus (1.12.2-1ubuntu1.2) ...\n",
            "Processing triggers for libc-bin (2.27-3ubuntu1.3) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/python3.7/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",
            "\n",
            "Processing triggers for systemd (237-3ubuntu10.52) ...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LKTWlO1dFNoP",
        "outputId": "1288c5dd-b397-4ff2-93c7-a61f108425c8"
      },
      "source": [
        "# Check version.\n",
        "!docker --version\n",
        "!pwd"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Docker version 20.10.7, build 20.10.7-0ubuntu1~18.04.1\n",
            "/content/videos_background_substitution\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CeTSNzP3KFTB",
        "outputId": "79b62dff-7a14-445a-e0ff-ec4742844d8a"
      },
      "source": [
        "!umount -vl /sys/fs/cgroup/cpuset"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "umount: /sys/fs/cgroup/cpuset (cgroup) unmounted\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yJwkJE0ZKZv_",
        "outputId": "fd6731c1-b390-4775-ea4d-98afc440538a"
      },
      "source": [
        "%shell\n",
        "!mount -vt sysfs sysfs /sys -o rw,remount\n",
        "!mount -vt tmpfs tmpfs /sys/fs/cgroup -o rw,remount\n",
        "!mount -vt cgroup cgroup /sys/fs/cgroup/cpuset -o rw,remount,cpuset"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "mount: sysfs mounted on /sys.\n",
            "mount: tmpfs mounted on /sys/fs/cgroup.\n",
            "mount: /sys/fs/cgroup/cpuset: mount point not mounted or bad option.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "erVlbatvSN2n"
      },
      "source": [
        "#Build image\n",
        "%%shell\n",
        "set -x\n",
        "dockerd -b none --iptables=0 -l warn &\n",
        "for i in $(seq 5); do [ ! -S \"/var/run/docker.sock\" ] && sleep 2 || break; done\n",
        "docker info\n",
        "docker network ls\n",
        "docker build -t bgeditor .\n",
        "docker images\n",
        "docker run hello-world\n",
        "kill $(jobs -p)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CkoT4pN0rcBw"
      },
      "source": [
        "#run image\n",
        "!docker run -it bgeditor"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}