diff --git a/MultiSpecific.ipynb b/MultiSpecific.ipynb index 231f7cd..ef13d1c 100644 --- a/MultiSpecific.ipynb +++ b/MultiSpecific.ipynb @@ -1 +1 @@ -{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"MultiSpecific.ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyNw8SfPWhG77cf/e7YZd178"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","metadata":{"id":"7_gtFoV8BuRx"},"source":["This is an example of SimSwap on processing video with multiple faces with designated sources.\n","\n","Code path: https://github.com/neuralchen/SimSwap\n","Paper path: https://arxiv.org/pdf/2106.06340v1.pdf."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"0Y1RfpzsCAl9","executionInfo":{"status":"ok","timestamp":1625380781426,"user_tz":-480,"elapsed":586,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"2a897b34-72f1-4515-ac6f-2f0e2d4ea4f7"},"source":["## make sure you are using a runtime with GPU\n","## you can check at Runtime/Change runtime type in the top bar.\n","!nvidia-smi"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Sun Jul 4 06:39:39 2021 \n","+-----------------------------------------------------------------------------+\n","| NVIDIA-SMI 465.27 Driver Version: 460.32.03 CUDA Version: 11.2 |\n","|-------------------------------+----------------------+----------------------+\n","| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n","| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n","| | | MIG M. |\n","|===============================+======================+======================|\n","| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n","| N/A 45C P8 9W / 70W | 0MiB / 15109MiB | 0% Default |\n","| | | N/A |\n","+-------------------------------+----------------------+----------------------+\n"," \n","+-----------------------------------------------------------------------------+\n","| Processes: |\n","| GPU GI CI PID Type Process name GPU Memory |\n","| ID ID Usage |\n","|=============================================================================|\n","| No running processes found |\n","+-----------------------------------------------------------------------------+\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"0Qzzx2UpDkqw"},"source":["All file changes make by this notebook are temporary. \n","You can try to mount your own google drive to store files if you wang.\n"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"VA_4CeWZCHLP","executionInfo":{"status":"ok","timestamp":1625380786661,"user_tz":-480,"elapsed":4693,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"d0665552-be92-45b0-aab2-f84c619a51fb"},"source":["!git clone https://github.com/neuralchen/SimSwap\n","!cd SimSwap && git pull"],"execution_count":2,"outputs":[{"output_type":"stream","text":["Cloning into 'SimSwap'...\n","remote: Enumerating objects: 667, done.\u001b[K\n","remote: Counting objects: 100% (48/48), done.\u001b[K\n","remote: Compressing objects: 100% (35/35), done.\u001b[K\n","remote: Total 667 (delta 19), reused 28 (delta 13), pack-reused 619\u001b[K\n","Receiving objects: 100% (667/667), 132.14 MiB | 44.44 MiB/s, done.\n","Resolving deltas: 100% (292/292), done.\n","Already up to date.\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Y5K4au_UCkKn","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625380797906,"user_tz":-480,"elapsed":11253,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"7429f153-bc6d-48c2-eb3c-21f1f02fede9"},"source":["!pip install insightface==0.2.1 onnxruntime moviepy\n","!pip install googledrivedownloader\n","!pip install imageio==2.4.1"],"execution_count":3,"outputs":[{"output_type":"stream","text":["Collecting insightface==0.2.1\n"," Downloading https://files.pythonhosted.org/packages/ee/1e/6395bbe0db665f187c8e49266cda54fcf661f182192370d409423e4943e4/insightface-0.2.1-py2.py3-none-any.whl\n","Collecting onnxruntime\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/f9/76/3d0f8bb2776961c7335693df06eccf8d099e48fa6fb552c7546867192603/onnxruntime-1.8.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.5MB)\n","\u001b[K |████████████████████████████████| 4.5MB 37.5MB/s \n","\u001b[?25hRequirement already satisfied: moviepy in /usr/local/lib/python3.7/dist-packages (0.2.3.5)\n","Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (0.22.2.post1)\n","Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (2.23.0)\n","Collecting onnx\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/3f/9b/54c950d3256e27f970a83cd0504efb183a24312702deed0179453316dbd0/onnx-1.9.0-cp37-cp37m-manylinux2010_x86_64.whl (12.2MB)\n","\u001b[K |████████████████████████████████| 12.2MB 32.2MB/s \n","\u001b[?25hRequirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (3.2.2)\n","Requirement already satisfied: Pillow in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (7.1.2)\n","Requirement already satisfied: scikit-image in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (0.16.2)\n","Requirement already satisfied: opencv-python in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (4.1.2.30)\n","Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (4.41.1)\n","Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (1.4.1)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (1.19.5)\n","Requirement already satisfied: easydict in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (1.9)\n","Requirement already satisfied: flatbuffers in /usr/local/lib/python3.7/dist-packages (from onnxruntime) (1.12)\n","Requirement already satisfied: protobuf in /usr/local/lib/python3.7/dist-packages (from onnxruntime) (3.12.4)\n","Requirement already satisfied: imageio<3.0,>=2.1.2 in /usr/local/lib/python3.7/dist-packages (from moviepy) (2.4.1)\n","Requirement already satisfied: decorator<5.0,>=4.0.2 in /usr/local/lib/python3.7/dist-packages (from moviepy) (4.4.2)\n","Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->insightface==0.2.1) (1.0.1)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->insightface==0.2.1) (3.0.4)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->insightface==0.2.1) (1.24.3)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->insightface==0.2.1) (2021.5.30)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->insightface==0.2.1) (2.10)\n","Requirement already satisfied: typing-extensions>=3.6.2.1 in /usr/local/lib/python3.7/dist-packages (from onnx->insightface==0.2.1) (3.7.4.3)\n","Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from onnx->insightface==0.2.1) (1.15.0)\n","Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->insightface==0.2.1) (2.8.1)\n","Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->insightface==0.2.1) (0.10.0)\n","Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->insightface==0.2.1) (2.4.7)\n","Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->insightface==0.2.1) (1.3.1)\n","Requirement already satisfied: networkx>=2.0 in /usr/local/lib/python3.7/dist-packages (from scikit-image->insightface==0.2.1) (2.5.1)\n","Requirement already satisfied: PyWavelets>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from scikit-image->insightface==0.2.1) (1.1.1)\n","Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf->onnxruntime) (57.0.0)\n","Installing collected packages: onnx, insightface, onnxruntime\n","Successfully installed insightface-0.2.1 onnx-1.9.0 onnxruntime-1.8.0\n","Requirement already satisfied: googledrivedownloader in /usr/local/lib/python3.7/dist-packages (0.4)\n","Requirement already satisfied: imageio==2.4.1 in /usr/local/lib/python3.7/dist-packages (2.4.1)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from imageio==2.4.1) (1.19.5)\n","Requirement already satisfied: pillow in /usr/local/lib/python3.7/dist-packages (from imageio==2.4.1) (7.1.2)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"gQ7ZoIbLFCye","executionInfo":{"status":"ok","timestamp":1625380798405,"user_tz":-480,"elapsed":533,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"8448a0a3-a19e-44c2-a044-f4d3f9152e91"},"source":["import os\n","os.chdir(\"SimSwap\")\n","!ls"],"execution_count":4,"outputs":[{"output_type":"stream","text":[" crop_224\t simswaplogo\n"," data\t\t test_one_image.py\n"," demo_file\t test_video_swapmulti.py\n"," docs\t\t test_video_swap_multispecific.py\n"," insightface_func test_video_swapsingle.py\n"," LICENSE\t test_video_swapspecific.py\n"," models\t\t test_wholeimage_swapmulti.py\n"," options\t test_wholeimage_swap_multispecific.py\n"," output\t\t test_wholeimage_swapsingle.py\n"," README.md\t test_wholeimage_swapspecific.py\n","'SimSwap colab.ipynb' util\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"ZvGp-p0nOmKE","executionInfo":{"status":"ok","timestamp":1625380798407,"user_tz":-480,"elapsed":17,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}}},"source":["## You can upload filed manually\n","# from google.colab import drive\n","# drive.mount('/content/gdrive')"],"execution_count":5,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"gLti1J0pEFjJ","executionInfo":{"status":"ok","timestamp":1625380813268,"user_tz":-480,"elapsed":14876,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"99dc9306-9b9a-475d-cc7d-a3f423bd1e81"},"source":["from google_drive_downloader import GoogleDriveDownloader\n","\n","### it seems that google drive link may not be permenant, you can find this ID from our open url.\n","# GoogleDriveDownloader.download_file_from_google_drive(file_id='1TLNdIufzwesDbyr_nVTR7Zrx9oRHLM_N',\n","# dest_path='./arcface_model/arcface_checkpoint.tar')\n","# GoogleDriveDownloader.download_file_from_google_drive(file_id='1PXkRiBUYbu1xWpQyDEJvGKeqqUFthJcI',\n","# dest_path='./checkpoints.zip')\n","\n","!wget -P ./arcface_model https://github.com/neuralchen/SimSwap/releases/download/1.0/arcface_checkpoint.tar\n","!wget https://github.com/neuralchen/SimSwap/releases/download/1.0/checkpoints.zip\n","!unzip ./checkpoints.zip -d ./checkpoints\n","!wget -P ./parsing_model/checkpoint https://github.com/neuralchen/SimSwap/releases/download/1.0/79999_iter.pth"],"execution_count":6,"outputs":[{"output_type":"stream","text":["--2021-07-04 06:39:56-- https://github.com/neuralchen/SimSwap/releases/download/1.0/arcface_checkpoint.tar\n","Resolving github.com (github.com)... 140.82.114.3\n","Connecting to github.com (github.com)|140.82.114.3|:443... connected.\n","HTTP request sent, awaiting response... 302 Found\n","Location: https://github-releases.githubusercontent.com/374891081/e17b9d00-dcb8-11eb-8c4f-1412bcea78a6?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20210704%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20210704T063956Z&X-Amz-Expires=300&X-Amz-Signature=b6d431c65405e894ddc994061c5fe8fe87db4e71e702513aec01f398a1004825&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=374891081&response-content-disposition=attachment%3B%20filename%3Darcface_checkpoint.tar&response-content-type=application%2Foctet-stream [following]\n","--2021-07-04 06:39:56-- https://github-releases.githubusercontent.com/374891081/e17b9d00-dcb8-11eb-8c4f-1412bcea78a6?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20210704%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20210704T063956Z&X-Amz-Expires=300&X-Amz-Signature=b6d431c65405e894ddc994061c5fe8fe87db4e71e702513aec01f398a1004825&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=374891081&response-content-disposition=attachment%3B%20filename%3Darcface_checkpoint.tar&response-content-type=application%2Foctet-stream\n","Resolving github-releases.githubusercontent.com (github-releases.githubusercontent.com)... 185.199.108.154, 185.199.109.154, 185.199.110.154, ...\n","Connecting to github-releases.githubusercontent.com (github-releases.githubusercontent.com)|185.199.108.154|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 766871429 (731M) [application/octet-stream]\n","Saving to: ‘./arcface_model/arcface_checkpoint.tar’\n","\n","arcface_checkpoint. 100%[===================>] 731.34M 64.4MB/s in 11s \n","\n","2021-07-04 06:40:07 (68.4 MB/s) - ‘./arcface_model/arcface_checkpoint.tar’ saved [766871429/766871429]\n","\n","--2021-07-04 06:40:07-- https://github.com/neuralchen/SimSwap/releases/download/1.0/checkpoints.zip\n","Resolving github.com (github.com)... 140.82.113.3\n","Connecting to github.com (github.com)|140.82.113.3|:443... connected.\n","HTTP request sent, awaiting response... 302 Found\n","Location: https://github-releases.githubusercontent.com/374891081/a8dac400-dcb6-11eb-933f-977cd7f5f554?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20210704%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20210704T063831Z&X-Amz-Expires=300&X-Amz-Signature=3fd2850d03abb9301bf5ba5969d82eb73cb0b940b85e45de2e1e34f1ba2eaf09&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=374891081&response-content-disposition=attachment%3B%20filename%3Dcheckpoints.zip&response-content-type=application%2Foctet-stream [following]\n","--2021-07-04 06:40:07-- https://github-releases.githubusercontent.com/374891081/a8dac400-dcb6-11eb-933f-977cd7f5f554?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20210704%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20210704T063831Z&X-Amz-Expires=300&X-Amz-Signature=3fd2850d03abb9301bf5ba5969d82eb73cb0b940b85e45de2e1e34f1ba2eaf09&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=374891081&response-content-disposition=attachment%3B%20filename%3Dcheckpoints.zip&response-content-type=application%2Foctet-stream\n","Resolving github-releases.githubusercontent.com (github-releases.githubusercontent.com)... 185.199.108.154, 185.199.109.154, 185.199.110.154, ...\n","Connecting to github-releases.githubusercontent.com (github-releases.githubusercontent.com)|185.199.108.154|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 256461775 (245M) [application/octet-stream]\n","Saving to: ‘checkpoints.zip’\n","\n","checkpoints.zip 100%[===================>] 244.58M 219MB/s in 1.1s \n","\n","2021-07-04 06:40:08 (219 MB/s) - ‘checkpoints.zip’ saved [256461775/256461775]\n","\n","Archive: ./checkpoints.zip\n"," creating: ./checkpoints/people/\n"," inflating: ./checkpoints/people/iter.txt \n"," inflating: ./checkpoints/people/latest_net_D1.pth \n"," inflating: ./checkpoints/people/latest_net_D2.pth \n"," inflating: ./checkpoints/people/latest_net_G.pth \n"," inflating: ./checkpoints/people/loss_log.txt \n"," inflating: ./checkpoints/people/opt.txt \n"," creating: ./checkpoints/people/web/\n"," creating: ./checkpoints/people/web/images/\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"fJ9DYRrCPIUL","executionInfo":{"status":"ok","timestamp":1625380821122,"user_tz":-480,"elapsed":7869,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"a3d1d841-440c-4244-8045-cb0ce3cc81fd"},"source":["!wget --no-check-certificate \"https://sh23tw.dm.files.1drv.com/y4mmGiIkNVigkSwOKDcV3nwMJulRGhbtHdkheehR5TArc52UjudUYNXAEvKCii2O5LAmzGCGK6IfleocxuDeoKxDZkNzDRSt4ZUlEt8GlSOpCXAFEkBwaZimtWGDRbpIGpb_pz9Nq5jATBQpezBS6G_UtspWTkgrXHHxhviV2nWy8APPx134zOZrUIbkSF6xnsqzs3uZ_SEX_m9Rey0ykpx9w\" -O antelope.zip\n","!unzip ./antelope.zip -d ./insightface_func/models/"],"execution_count":7,"outputs":[{"output_type":"stream","text":["--2021-07-04 06:40:11-- https://sh23tw.dm.files.1drv.com/y4mmGiIkNVigkSwOKDcV3nwMJulRGhbtHdkheehR5TArc52UjudUYNXAEvKCii2O5LAmzGCGK6IfleocxuDeoKxDZkNzDRSt4ZUlEt8GlSOpCXAFEkBwaZimtWGDRbpIGpb_pz9Nq5jATBQpezBS6G_UtspWTkgrXHHxhviV2nWy8APPx134zOZrUIbkSF6xnsqzs3uZ_SEX_m9Rey0ykpx9w\n","Resolving sh23tw.dm.files.1drv.com (sh23tw.dm.files.1drv.com)... 13.107.42.12\n","Connecting to sh23tw.dm.files.1drv.com (sh23tw.dm.files.1drv.com)|13.107.42.12|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 248024513 (237M) [application/zip]\n","Saving to: ‘antelope.zip’\n","\n","antelope.zip 100%[===================>] 236.53M 52.4MB/s in 4.7s \n","\n","2021-07-04 06:40:16 (49.9 MB/s) - ‘antelope.zip’ saved [248024513/248024513]\n","\n","Archive: ./antelope.zip\n"," creating: ./insightface_func/models/antelope/\n"," inflating: ./insightface_func/models/antelope/glintr100.onnx \n"," inflating: ./insightface_func/models/antelope/scrfd_10g_bnkps.onnx \n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"PfSsND36EMvn","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625380827902,"user_tz":-480,"elapsed":6811,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"8130e97d-b4a0-4988-85fb-e2bc3e755259"},"source":["import cv2\n","import torch\n","import fractions\n","import numpy as np\n","from PIL import Image\n","import torch.nn.functional as F\n","from torchvision import transforms\n","from models.models import create_model\n","from options.test_options import TestOptions\n","from insightface_func.face_detect_crop_multi import Face_detect_crop\n","from util.videoswap_multispecific import video_swap\n","import os\n","import glob"],"execution_count":8,"outputs":[{"output_type":"stream","text":["Imageio: 'ffmpeg-linux64-v3.3.1' was not found on your computer; downloading it now.\n","Try 1. Download from https://github.com/imageio/imageio-binaries/raw/master/ffmpeg/ffmpeg-linux64-v3.3.1 (43.8 MB)\n","Downloading: 8192/45929032 bytes (0.0%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b3432448/45929032 bytes (7.5%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b7036928/45929032 bytes (15.3%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b10641408/45929032 bytes (23.2%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b14278656/45929032 bytes (31.1%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b18104320/45929032 bytes (39.4%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b21954560/45929032 bytes (47.8%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b25780224/45929032 bytes (56.1%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b29736960/45929032 bytes (64.7%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b33488896/45929032 bytes (72.9%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b37093376/45929032 bytes (80.8%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b40689664/45929032 bytes (88.6%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b44392448/45929032 bytes (96.7%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b45929032/45929032 bytes (100.0%)\n"," Done\n","File saved as /root/.imageio/ffmpeg/ffmpeg-linux64-v3.3.1.\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"rxSbZ2EDNDlf","executionInfo":{"status":"ok","timestamp":1625380827903,"user_tz":-480,"elapsed":12,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}}},"source":["def lcm(a, b): return abs(a * b) / fractions.gcd(a, b) if a and b else 0\n","\n","transformer = transforms.Compose([\n"," transforms.ToTensor(),\n"," #transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n"," ])\n","\n","transformer_Arcface = transforms.Compose([\n"," transforms.ToTensor(),\n"," transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n"," ])\n"],"execution_count":9,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ye8iS0UVPMRg","executionInfo":{"status":"ok","timestamp":1625380828574,"user_tz":-480,"elapsed":680,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"cb5a4b02-b1d0-4ff8-f542-5c1b3c9703d9"},"source":["!ls ./checkpoints"],"execution_count":10,"outputs":[{"output_type":"stream","text":["people\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"wwJOwR9LNKRz","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625380828576,"user_tz":-480,"elapsed":13,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"0f92f785-4d9c-4130-b24d-76871b2dafba"},"source":["opt = TestOptions()\n","opt.initialize()\n","opt.parser.add_argument('-f') ## dummy arg to avoid bug\n","opt = opt.parse()\n","opt.multisepcific_dir = './demo_file/multispecific' ## or replace it with folder from your own google drive\n"," ## and remember to follow the dir structure in usage.md\n","opt.video_path = './demo_file/multi_people_1080p.mp4' ## or replace it with video from your own google drive\n","opt.output_path = './output/multi_test_multispecific.mp4'\n","opt.temp_path = './tmp'\n","opt.Arc_path = './arcface_model/arcface_checkpoint.tar'\n","opt.name = 'people'\n","opt.isTrain = False\n","opt.use_mask = True ## new feature up-to-date\n","\n","crop_size = 224\n"],"execution_count":11,"outputs":[{"output_type":"stream","text":["------------ Options -------------\n","Arc_path: models/BEST_checkpoint.tar\n","aspect_ratio: 1.0\n","batchSize: 8\n","checkpoints_dir: ./checkpoints\n","cluster_path: features_clustered_010.npy\n","data_type: 32\n","dataroot: ./datasets/cityscapes/\n","display_winsize: 512\n","engine: None\n","export_onnx: None\n","f: /root/.local/share/jupyter/runtime/kernel-19937219-895d-4d02-9a72-5cfa0e889adf.json\n","feat_num: 3\n","fineSize: 512\n","fp16: False\n","gpu_ids: [0]\n","how_many: 50\n","id_thres: 0.03\n","image_size: 224\n","input_nc: 3\n","instance_feat: False\n","isTrain: False\n","label_feat: False\n","label_nc: 0\n","latent_size: 512\n","loadSize: 1024\n","load_features: False\n","local_rank: 0\n","max_dataset_size: inf\n","model: pix2pixHD\n","multisepcific_dir: ./demo_file/multispecific\n","nThreads: 2\n","n_blocks_global: 6\n","n_blocks_local: 3\n","n_clusters: 10\n","n_downsample_E: 4\n","n_downsample_global: 3\n","n_local_enhancers: 1\n","name: people\n","nef: 16\n","netG: global\n","ngf: 64\n","niter_fix_global: 0\n","no_flip: False\n","no_instance: False\n","no_simswaplogo: False\n","norm: batch\n","norm_G: spectralspadesyncbatch3x3\n","ntest: inf\n","onnx: None\n","output_nc: 3\n","output_path: ./output/\n","phase: test\n","pic_a_path: ./crop_224/gdg.jpg\n","pic_b_path: ./crop_224/zrf.jpg\n","pic_specific_path: ./crop_224/zrf.jpg\n","resize_or_crop: scale_width\n","results_dir: ./results/\n","semantic_nc: 3\n","serial_batches: False\n","temp_path: ./temp_results\n","tf_log: False\n","use_dropout: False\n","use_encoded_image: False\n","verbose: False\n","video_path: ./demo_file/multi_people_1080p.mp4\n","which_epoch: latest\n","-------------- End ----------------\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"UFt8zQrAMq9F","executionInfo":{"status":"ok","timestamp":1625381428564,"user_tz":-480,"elapsed":599996,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"46188013-3f6d-4174-9959-d1fd203dcc0d"},"source":["pic_specific = opt.pic_specific_path\n","crop_size = 224\n","multisepcific_dir = opt.multisepcific_dir\n","\n","torch.nn.Module.dump_patches = True\n","model = create_model(opt)\n","model.eval()\n","\n","app = Face_detect_crop(name='antelope', root='./insightface_func/models')\n","app.prepare(ctx_id= 0, det_thresh=0.6, det_size=(640,640))\n","# The specific person to be swapped(source)\n","source_specific_id_nonorm_list = []\n","source_path = os.path.join(multisepcific_dir,'SRC_*')\n","source_specific_images_path = sorted(glob.glob(source_path))\n","\n","with torch.no_grad():\n"," for source_specific_image_path in source_specific_images_path:\n"," specific_person_whole = cv2.imread(source_specific_image_path)\n"," specific_person_align_crop, _ = app.get(specific_person_whole,crop_size)\n"," specific_person_align_crop_pil = Image.fromarray(cv2.cvtColor(specific_person_align_crop[0],cv2.COLOR_BGR2RGB)) \n"," specific_person = transformer_Arcface(specific_person_align_crop_pil)\n"," specific_person = specific_person.view(-1, specific_person.shape[0], specific_person.shape[1], specific_person.shape [2])\n"," # convert numpy to tensor\n"," specific_person = specific_person.cuda()\n"," #create latent id\n"," specific_person_downsample = F.interpolate(specific_person, scale_factor=0.5)\n"," specific_person_id_nonorm = model.netArc(specific_person_downsample)\n"," source_specific_id_nonorm_list.append(specific_person_id_nonorm.clone())\n","\n"," # The person who provides id information (list)\n"," target_id_norm_list = []\n"," target_path = os.path.join(multisepcific_dir,'DST_*')\n"," target_images_path = sorted(glob.glob(target_path))\n","\n"," for target_image_path in target_images_path:\n"," img_a_whole = cv2.imread(target_image_path)\n"," img_a_align_crop, _ = app.get(img_a_whole,crop_size)\n"," img_a_align_crop_pil = Image.fromarray(cv2.cvtColor(img_a_align_crop[0],cv2.COLOR_BGR2RGB)) \n"," img_a = transformer_Arcface(img_a_align_crop_pil)\n"," img_id = img_a.view(-1, img_a.shape[0], img_a.shape[1], img_a.shape[2])\n"," # convert numpy to tensor\n"," img_id = img_id.cuda()\n"," #create latent id\n"," img_id_downsample = F.interpolate(img_id, scale_factor=0.5)\n"," latend_id = model.netArc(img_id_downsample)\n"," latend_id = F.normalize(latend_id, p=2, dim=1)\n"," target_id_norm_list.append(latend_id.clone())\n"," \n"," assert len(target_id_norm_list) == len(source_specific_id_nonorm_list), \"The number of images in source and target directory must be same !!!\"\n"," video_swap(opt.video_path, target_id_norm_list,source_specific_id_nonorm_list, opt.id_thres, \\\n"," model, app, opt.output_path,temp_results_dir=opt.temp_path,no_simswaplogo=opt.no_simswaplogo,use_mask=opt.use_mask)"],"execution_count":12,"outputs":[{"output_type":"stream","text":["input mean and std: 127.5 127.5\n","find model: ./insightface_func/models/antelope/glintr100.onnx recognition\n","find model: ./insightface_func/models/antelope/scrfd_10g_bnkps.onnx detection\n","set det-size: (640, 640)\n"],"name":"stdout"},{"output_type":"stream","text":["\r 0%| | 0/594 [00:00>>> Building video ./output/multi_test_multispecific.mp4\n","[MoviePy] Writing audio in multi_test_multispecificTEMP_MPY_wvf_snd.mp3\n"],"name":"stdout"},{"output_type":"stream","text":["100%|██████████| 438/438 [00:00<00:00, 832.16it/s]"],"name":"stderr"},{"output_type":"stream","text":["[MoviePy] Done.\n","[MoviePy] Writing video ./output/multi_test_multispecific.mp4\n"],"name":"stdout"},{"output_type":"stream","text":["\n","100%|██████████| 595/595 [00:53<00:00, 11.11it/s]\n"],"name":"stderr"},{"output_type":"stream","text":["[MoviePy] Done.\n","[MoviePy] >>>> Video ready: ./output/multi_test_multispecific.mp4 \n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Rty2GsyZZrI6"},"source":[],"execution_count":null,"outputs":[]}]} \ No newline at end of file +{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"MultiSpecific.ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyNw8SfPWhG77cf/e7YZd178"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","metadata":{"id":"7_gtFoV8BuRx"},"source":["This is an example of SimSwap on processing video with multiple faces with designated sources.\n","\n","Code path: https://github.com/neuralchen/SimSwap\n","Paper path: https://arxiv.org/pdf/2106.06340v1.pdf."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"0Y1RfpzsCAl9","executionInfo":{"status":"ok","timestamp":1625380781426,"user_tz":-480,"elapsed":586,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"2a897b34-72f1-4515-ac6f-2f0e2d4ea4f7"},"source":["## make sure you are using a runtime with GPU\n","## you can check at Runtime/Change runtime type in the top bar.\n","!nvidia-smi"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Sun Jul 4 06:39:39 2021 \n","+-----------------------------------------------------------------------------+\n","| NVIDIA-SMI 465.27 Driver Version: 460.32.03 CUDA Version: 11.2 |\n","|-------------------------------+----------------------+----------------------+\n","| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n","| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n","| | | MIG M. |\n","|===============================+======================+======================|\n","| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n","| N/A 45C P8 9W / 70W | 0MiB / 15109MiB | 0% Default |\n","| | | N/A |\n","+-------------------------------+----------------------+----------------------+\n"," \n","+-----------------------------------------------------------------------------+\n","| Processes: |\n","| GPU GI CI PID Type Process name GPU Memory |\n","| ID ID Usage |\n","|=============================================================================|\n","| No running processes found |\n","+-----------------------------------------------------------------------------+\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"0Qzzx2UpDkqw"},"source":["All file changes make by this notebook are temporary. \n","You can try to mount your own google drive to store files if you wang.\n"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"VA_4CeWZCHLP","executionInfo":{"status":"ok","timestamp":1625380786661,"user_tz":-480,"elapsed":4693,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"d0665552-be92-45b0-aab2-f84c619a51fb"},"source":["!git clone https://github.com/neuralchen/SimSwap\n","!cd SimSwap && git pull"],"execution_count":2,"outputs":[{"output_type":"stream","text":["Cloning into 'SimSwap'...\n","remote: Enumerating objects: 667, done.\u001b[K\n","remote: Counting objects: 100% (48/48), done.\u001b[K\n","remote: Compressing objects: 100% (35/35), done.\u001b[K\n","remote: Total 667 (delta 19), reused 28 (delta 13), pack-reused 619\u001b[K\n","Receiving objects: 100% (667/667), 132.14 MiB | 44.44 MiB/s, done.\n","Resolving deltas: 100% (292/292), done.\n","Already up to date.\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Y5K4au_UCkKn","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625380797906,"user_tz":-480,"elapsed":11253,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"7429f153-bc6d-48c2-eb3c-21f1f02fede9"},"source":["!pip install insightface==0.2.1 onnxruntime moviepy\n","!pip install googledrivedownloader\n","!pip install imageio==2.4.1"],"execution_count":3,"outputs":[{"output_type":"stream","text":["Collecting insightface==0.2.1\n"," Downloading https://files.pythonhosted.org/packages/ee/1e/6395bbe0db665f187c8e49266cda54fcf661f182192370d409423e4943e4/insightface-0.2.1-py2.py3-none-any.whl\n","Collecting onnxruntime\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/f9/76/3d0f8bb2776961c7335693df06eccf8d099e48fa6fb552c7546867192603/onnxruntime-1.8.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.5MB)\n","\u001b[K |████████████████████████████████| 4.5MB 37.5MB/s \n","\u001b[?25hRequirement already satisfied: moviepy in /usr/local/lib/python3.7/dist-packages (0.2.3.5)\n","Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (0.22.2.post1)\n","Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (2.23.0)\n","Collecting onnx\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/3f/9b/54c950d3256e27f970a83cd0504efb183a24312702deed0179453316dbd0/onnx-1.9.0-cp37-cp37m-manylinux2010_x86_64.whl (12.2MB)\n","\u001b[K |████████████████████████████████| 12.2MB 32.2MB/s \n","\u001b[?25hRequirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (3.2.2)\n","Requirement already satisfied: Pillow in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (7.1.2)\n","Requirement already satisfied: scikit-image in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (0.16.2)\n","Requirement already satisfied: opencv-python in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (4.1.2.30)\n","Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (4.41.1)\n","Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (1.4.1)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (1.19.5)\n","Requirement already satisfied: easydict in /usr/local/lib/python3.7/dist-packages (from insightface==0.2.1) (1.9)\n","Requirement already satisfied: flatbuffers in /usr/local/lib/python3.7/dist-packages (from onnxruntime) (1.12)\n","Requirement already satisfied: protobuf in /usr/local/lib/python3.7/dist-packages (from onnxruntime) (3.12.4)\n","Requirement already satisfied: imageio<3.0,>=2.1.2 in /usr/local/lib/python3.7/dist-packages (from moviepy) (2.4.1)\n","Requirement already satisfied: decorator<5.0,>=4.0.2 in /usr/local/lib/python3.7/dist-packages (from moviepy) (4.4.2)\n","Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->insightface==0.2.1) (1.0.1)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->insightface==0.2.1) (3.0.4)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->insightface==0.2.1) (1.24.3)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->insightface==0.2.1) (2021.5.30)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->insightface==0.2.1) (2.10)\n","Requirement already satisfied: typing-extensions>=3.6.2.1 in /usr/local/lib/python3.7/dist-packages (from onnx->insightface==0.2.1) (3.7.4.3)\n","Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from onnx->insightface==0.2.1) (1.15.0)\n","Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->insightface==0.2.1) (2.8.1)\n","Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->insightface==0.2.1) (0.10.0)\n","Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->insightface==0.2.1) (2.4.7)\n","Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->insightface==0.2.1) (1.3.1)\n","Requirement already satisfied: networkx>=2.0 in /usr/local/lib/python3.7/dist-packages (from scikit-image->insightface==0.2.1) (2.5.1)\n","Requirement already satisfied: PyWavelets>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from scikit-image->insightface==0.2.1) (1.1.1)\n","Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf->onnxruntime) (57.0.0)\n","Installing collected packages: onnx, insightface, onnxruntime\n","Successfully installed insightface-0.2.1 onnx-1.9.0 onnxruntime-1.8.0\n","Requirement already satisfied: googledrivedownloader in /usr/local/lib/python3.7/dist-packages (0.4)\n","Requirement already satisfied: imageio==2.4.1 in /usr/local/lib/python3.7/dist-packages (2.4.1)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from imageio==2.4.1) (1.19.5)\n","Requirement already satisfied: pillow in /usr/local/lib/python3.7/dist-packages (from imageio==2.4.1) (7.1.2)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"gQ7ZoIbLFCye","executionInfo":{"status":"ok","timestamp":1625380798405,"user_tz":-480,"elapsed":533,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"8448a0a3-a19e-44c2-a044-f4d3f9152e91"},"source":["import os\n","os.chdir(\"SimSwap\")\n","!ls"],"execution_count":4,"outputs":[{"output_type":"stream","text":[" crop_224\t simswaplogo\n"," data\t\t test_one_image.py\n"," demo_file\t test_video_swapmulti.py\n"," docs\t\t test_video_swap_multispecific.py\n"," insightface_func test_video_swapsingle.py\n"," LICENSE\t test_video_swapspecific.py\n"," models\t\t test_wholeimage_swapmulti.py\n"," options\t test_wholeimage_swap_multispecific.py\n"," output\t\t test_wholeimage_swapsingle.py\n"," README.md\t test_wholeimage_swapspecific.py\n","'SimSwap colab.ipynb' util\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"ZvGp-p0nOmKE","executionInfo":{"status":"ok","timestamp":1625380798407,"user_tz":-480,"elapsed":17,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}}},"source":["## You can upload filed manually\n","# from google.colab import drive\n","# drive.mount('/content/gdrive')"],"execution_count":5,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"gLti1J0pEFjJ","executionInfo":{"status":"ok","timestamp":1625380813268,"user_tz":-480,"elapsed":14876,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"99dc9306-9b9a-475d-cc7d-a3f423bd1e81"},"source":["from google_drive_downloader import GoogleDriveDownloader\n","\n","### it seems that google drive link may not be permenant, you can find this ID from our open url.\n","# GoogleDriveDownloader.download_file_from_google_drive(file_id='1TLNdIufzwesDbyr_nVTR7Zrx9oRHLM_N',\n","# dest_path='./arcface_model/arcface_checkpoint.tar')\n","# GoogleDriveDownloader.download_file_from_google_drive(file_id='1PXkRiBUYbu1xWpQyDEJvGKeqqUFthJcI',\n","# dest_path='./checkpoints.zip')\n","\n","!wget -P ./arcface_model https://github.com/neuralchen/SimSwap/releases/download/1.0/arcface_checkpoint.tar\n","!wget https://github.com/neuralchen/SimSwap/releases/download/1.0/checkpoints.zip\n","!unzip ./checkpoints.zip -d ./checkpoints\n","!wget -P ./parsing_model/checkpoint https://github.com/neuralchen/SimSwap/releases/download/1.0/79999_iter.pth"],"execution_count":6,"outputs":[{"output_type":"stream","text":["--2021-07-04 06:39:56-- https://github.com/neuralchen/SimSwap/releases/download/1.0/arcface_checkpoint.tar\n","Resolving github.com (github.com)... 140.82.114.3\n","Connecting to github.com (github.com)|140.82.114.3|:443... connected.\n","HTTP request sent, awaiting response... 302 Found\n","Location: https://github-releases.githubusercontent.com/374891081/e17b9d00-dcb8-11eb-8c4f-1412bcea78a6?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20210704%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20210704T063956Z&X-Amz-Expires=300&X-Amz-Signature=b6d431c65405e894ddc994061c5fe8fe87db4e71e702513aec01f398a1004825&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=374891081&response-content-disposition=attachment%3B%20filename%3Darcface_checkpoint.tar&response-content-type=application%2Foctet-stream [following]\n","--2021-07-04 06:39:56-- https://github-releases.githubusercontent.com/374891081/e17b9d00-dcb8-11eb-8c4f-1412bcea78a6?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20210704%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20210704T063956Z&X-Amz-Expires=300&X-Amz-Signature=b6d431c65405e894ddc994061c5fe8fe87db4e71e702513aec01f398a1004825&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=374891081&response-content-disposition=attachment%3B%20filename%3Darcface_checkpoint.tar&response-content-type=application%2Foctet-stream\n","Resolving github-releases.githubusercontent.com (github-releases.githubusercontent.com)... 185.199.108.154, 185.199.109.154, 185.199.110.154, ...\n","Connecting to github-releases.githubusercontent.com (github-releases.githubusercontent.com)|185.199.108.154|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 766871429 (731M) [application/octet-stream]\n","Saving to: ‘./arcface_model/arcface_checkpoint.tar’\n","\n","arcface_checkpoint. 100%[===================>] 731.34M 64.4MB/s in 11s \n","\n","2021-07-04 06:40:07 (68.4 MB/s) - ‘./arcface_model/arcface_checkpoint.tar’ saved [766871429/766871429]\n","\n","--2021-07-04 06:40:07-- https://github.com/neuralchen/SimSwap/releases/download/1.0/checkpoints.zip\n","Resolving github.com (github.com)... 140.82.113.3\n","Connecting to github.com (github.com)|140.82.113.3|:443... connected.\n","HTTP request sent, awaiting response... 302 Found\n","Location: https://github-releases.githubusercontent.com/374891081/a8dac400-dcb6-11eb-933f-977cd7f5f554?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20210704%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20210704T063831Z&X-Amz-Expires=300&X-Amz-Signature=3fd2850d03abb9301bf5ba5969d82eb73cb0b940b85e45de2e1e34f1ba2eaf09&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=374891081&response-content-disposition=attachment%3B%20filename%3Dcheckpoints.zip&response-content-type=application%2Foctet-stream [following]\n","--2021-07-04 06:40:07-- https://github-releases.githubusercontent.com/374891081/a8dac400-dcb6-11eb-933f-977cd7f5f554?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20210704%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20210704T063831Z&X-Amz-Expires=300&X-Amz-Signature=3fd2850d03abb9301bf5ba5969d82eb73cb0b940b85e45de2e1e34f1ba2eaf09&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=374891081&response-content-disposition=attachment%3B%20filename%3Dcheckpoints.zip&response-content-type=application%2Foctet-stream\n","Resolving github-releases.githubusercontent.com (github-releases.githubusercontent.com)... 185.199.108.154, 185.199.109.154, 185.199.110.154, ...\n","Connecting to github-releases.githubusercontent.com (github-releases.githubusercontent.com)|185.199.108.154|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 256461775 (245M) [application/octet-stream]\n","Saving to: ‘checkpoints.zip’\n","\n","checkpoints.zip 100%[===================>] 244.58M 219MB/s in 1.1s \n","\n","2021-07-04 06:40:08 (219 MB/s) - ‘checkpoints.zip’ saved [256461775/256461775]\n","\n","Archive: ./checkpoints.zip\n"," creating: ./checkpoints/people/\n"," inflating: ./checkpoints/people/iter.txt \n"," inflating: ./checkpoints/people/latest_net_D1.pth \n"," inflating: ./checkpoints/people/latest_net_D2.pth \n"," inflating: ./checkpoints/people/latest_net_G.pth \n"," inflating: ./checkpoints/people/loss_log.txt \n"," inflating: ./checkpoints/people/opt.txt \n"," creating: ./checkpoints/people/web/\n"," creating: ./checkpoints/people/web/images/\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"fJ9DYRrCPIUL","executionInfo":{"status":"ok","timestamp":1625380821122,"user_tz":-480,"elapsed":7869,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"a3d1d841-440c-4244-8045-cb0ce3cc81fd"},"source":["!wget --no-check-certificate \"https://sh23tw.dm.files.1drv.com/y4mmGiIkNVigkSwOKDcV3nwMJulRGhbtHdkheehR5TArc52UjudUYNXAEvKCii2O5LAmzGCGK6IfleocxuDeoKxDZkNzDRSt4ZUlEt8GlSOpCXAFEkBwaZimtWGDRbpIGpb_pz9Nq5jATBQpezBS6G_UtspWTkgrXHHxhviV2nWy8APPx134zOZrUIbkSF6xnsqzs3uZ_SEX_m9Rey0ykpx9w\" -O antelope.zip\n","!unzip ./antelope.zip -d ./insightface_func/models/"],"execution_count":7,"outputs":[{"output_type":"stream","text":["--2021-07-04 06:40:11-- https://sh23tw.dm.files.1drv.com/y4mmGiIkNVigkSwOKDcV3nwMJulRGhbtHdkheehR5TArc52UjudUYNXAEvKCii2O5LAmzGCGK6IfleocxuDeoKxDZkNzDRSt4ZUlEt8GlSOpCXAFEkBwaZimtWGDRbpIGpb_pz9Nq5jATBQpezBS6G_UtspWTkgrXHHxhviV2nWy8APPx134zOZrUIbkSF6xnsqzs3uZ_SEX_m9Rey0ykpx9w\n","Resolving sh23tw.dm.files.1drv.com (sh23tw.dm.files.1drv.com)... 13.107.42.12\n","Connecting to sh23tw.dm.files.1drv.com (sh23tw.dm.files.1drv.com)|13.107.42.12|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 248024513 (237M) [application/zip]\n","Saving to: ‘antelope.zip’\n","\n","antelope.zip 100%[===================>] 236.53M 52.4MB/s in 4.7s \n","\n","2021-07-04 06:40:16 (49.9 MB/s) - ‘antelope.zip’ saved [248024513/248024513]\n","\n","Archive: ./antelope.zip\n"," creating: ./insightface_func/models/antelope/\n"," inflating: ./insightface_func/models/antelope/glintr100.onnx \n"," inflating: ./insightface_func/models/antelope/scrfd_10g_bnkps.onnx \n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"PfSsND36EMvn","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625380827902,"user_tz":-480,"elapsed":6811,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"8130e97d-b4a0-4988-85fb-e2bc3e755259"},"source":["import cv2\n","import torch\n","import fractions\n","import numpy as np\n","from PIL import Image\n","import torch.nn.functional as F\n","from torchvision import transforms\n","from models.models import create_model\n","from options.test_options import TestOptions\n","from insightface_func.face_detect_crop_multi import Face_detect_crop\n","from util.videoswap_multispecific import video_swap\n","import os\n","import glob"],"execution_count":8,"outputs":[{"output_type":"stream","text":["Imageio: 'ffmpeg-linux64-v3.3.1' was not found on your computer; downloading it now.\n","Try 1. Download from https://github.com/imageio/imageio-binaries/raw/master/ffmpeg/ffmpeg-linux64-v3.3.1 (43.8 MB)\n","Downloading: 8192/45929032 bytes (0.0%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b3432448/45929032 bytes (7.5%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b7036928/45929032 bytes (15.3%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b10641408/45929032 bytes (23.2%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b14278656/45929032 bytes (31.1%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b18104320/45929032 bytes (39.4%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b21954560/45929032 bytes (47.8%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b25780224/45929032 bytes (56.1%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b29736960/45929032 bytes (64.7%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b33488896/45929032 bytes (72.9%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b37093376/45929032 bytes (80.8%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b40689664/45929032 bytes (88.6%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b44392448/45929032 bytes (96.7%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b45929032/45929032 bytes (100.0%)\n"," Done\n","File saved as /root/.imageio/ffmpeg/ffmpeg-linux64-v3.3.1.\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"rxSbZ2EDNDlf","executionInfo":{"status":"ok","timestamp":1625380827903,"user_tz":-480,"elapsed":12,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}}},"source":["def lcm(a, b): return abs(a * b) / fractions.gcd(a, b) if a and b else 0\n","\n","transformer = transforms.Compose([\n"," transforms.ToTensor(),\n"," #transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n"," ])\n","\n","transformer_Arcface = transforms.Compose([\n"," transforms.ToTensor(),\n"," transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n"," ])\n"],"execution_count":9,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ye8iS0UVPMRg","executionInfo":{"status":"ok","timestamp":1625380828574,"user_tz":-480,"elapsed":680,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"cb5a4b02-b1d0-4ff8-f542-5c1b3c9703d9"},"source":["!ls ./checkpoints"],"execution_count":10,"outputs":[{"output_type":"stream","text":["people\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"wwJOwR9LNKRz","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625380828576,"user_tz":-480,"elapsed":13,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"0f92f785-4d9c-4130-b24d-76871b2dafba"},"source":["opt = TestOptions()\n","opt.initialize()\n","opt.parser.add_argument('-f') ## dummy arg to avoid bug\n","opt = opt.parse()\n","opt.multisepcific_dir = './demo_file/multispecific' ## or replace it with folder from your own google drive\n"," ## and remember to follow the dir structure in usage.md\n","opt.video_path = './demo_file/multi_people_1080p.mp4' ## or replace it with video from your own google drive\n","opt.output_path = './output/multi_test_multispecific.mp4'\n","opt.temp_path = './tmp'\n","opt.Arc_path = './arcface_model/arcface_checkpoint.tar'\n","opt.name = 'people'\n","opt.isTrain = False\n","opt.use_mask = True ## new feature up-to-date\n","\n","crop_size = opt.crop_size\n"],"execution_count":11,"outputs":[{"output_type":"stream","text":["------------ Options -------------\n","Arc_path: models/BEST_checkpoint.tar\n","aspect_ratio: 1.0\n","batchSize: 8\n","checkpoints_dir: ./checkpoints\n","cluster_path: features_clustered_010.npy\n","data_type: 32\n","dataroot: ./datasets/cityscapes/\n","display_winsize: 512\n","engine: None\n","export_onnx: None\n","f: /root/.local/share/jupyter/runtime/kernel-19937219-895d-4d02-9a72-5cfa0e889adf.json\n","feat_num: 3\n","fineSize: 512\n","fp16: False\n","gpu_ids: [0]\n","how_many: 50\n","id_thres: 0.03\n","image_size: 224\n","input_nc: 3\n","instance_feat: False\n","isTrain: False\n","label_feat: False\n","label_nc: 0\n","latent_size: 512\n","loadSize: 1024\n","load_features: False\n","local_rank: 0\n","max_dataset_size: inf\n","model: pix2pixHD\n","multisepcific_dir: ./demo_file/multispecific\n","nThreads: 2\n","n_blocks_global: 6\n","n_blocks_local: 3\n","n_clusters: 10\n","n_downsample_E: 4\n","n_downsample_global: 3\n","n_local_enhancers: 1\n","name: people\n","nef: 16\n","netG: global\n","ngf: 64\n","niter_fix_global: 0\n","no_flip: False\n","no_instance: False\n","no_simswaplogo: False\n","norm: batch\n","norm_G: spectralspadesyncbatch3x3\n","ntest: inf\n","onnx: None\n","output_nc: 3\n","output_path: ./output/\n","phase: test\n","pic_a_path: ./crop_224/gdg.jpg\n","pic_b_path: ./crop_224/zrf.jpg\n","pic_specific_path: ./crop_224/zrf.jpg\n","resize_or_crop: scale_width\n","results_dir: ./results/\n","semantic_nc: 3\n","serial_batches: False\n","temp_path: ./temp_results\n","tf_log: False\n","use_dropout: False\n","use_encoded_image: False\n","verbose: False\n","video_path: ./demo_file/multi_people_1080p.mp4\n","which_epoch: latest\n","-------------- End ----------------\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"UFt8zQrAMq9F","executionInfo":{"status":"ok","timestamp":1625381428564,"user_tz":-480,"elapsed":599996,"user":{"displayName":"José Lampreia","photoUrl":"","userId":"16015278604201270582"}},"outputId":"46188013-3f6d-4174-9959-d1fd203dcc0d"},"source":["pic_specific = opt.pic_specific_path\n","crop_size = opt.crop_size\n","multisepcific_dir = opt.multisepcific_dir\n","\n","torch.nn.Module.dump_patches = True\n","model = create_model(opt)\n","model.eval()\n","\n","app = Face_detect_crop(name='antelope', root='./insightface_func/models')\n","app.prepare(ctx_id= 0, det_thresh=0.6, det_size=(640,640))\n","# The specific person to be swapped(source)\n","source_specific_id_nonorm_list = []\n","source_path = os.path.join(multisepcific_dir,'SRC_*')\n","source_specific_images_path = sorted(glob.glob(source_path))\n","\n","with torch.no_grad():\n"," for source_specific_image_path in source_specific_images_path:\n"," specific_person_whole = cv2.imread(source_specific_image_path)\n"," specific_person_align_crop, _ = app.get(specific_person_whole,crop_size)\n"," specific_person_align_crop_pil = Image.fromarray(cv2.cvtColor(specific_person_align_crop[0],cv2.COLOR_BGR2RGB)) \n"," specific_person = transformer_Arcface(specific_person_align_crop_pil)\n"," specific_person = specific_person.view(-1, specific_person.shape[0], specific_person.shape[1], specific_person.shape [2])\n"," # convert numpy to tensor\n"," specific_person = specific_person.cuda()\n"," #create latent id\n"," specific_person_downsample = F.interpolate(specific_person, size=(112,112))\n"," specific_person_id_nonorm = model.netArc(specific_person_downsample)\n"," source_specific_id_nonorm_list.append(specific_person_id_nonorm.clone())\n","\n"," # The person who provides id information (list)\n"," target_id_norm_list = []\n"," target_path = os.path.join(multisepcific_dir,'DST_*')\n"," target_images_path = sorted(glob.glob(target_path))\n","\n"," for target_image_path in target_images_path:\n"," img_a_whole = cv2.imread(target_image_path)\n"," img_a_align_crop, _ = app.get(img_a_whole,crop_size)\n"," img_a_align_crop_pil = Image.fromarray(cv2.cvtColor(img_a_align_crop[0],cv2.COLOR_BGR2RGB)) \n"," img_a = transformer_Arcface(img_a_align_crop_pil)\n"," img_id = img_a.view(-1, img_a.shape[0], img_a.shape[1], img_a.shape[2])\n"," # convert numpy to tensor\n"," img_id = img_id.cuda()\n"," #create latent id\n"," img_id_downsample = F.interpolate(img_id, size=(112,112))\n"," latend_id = model.netArc(img_id_downsample)\n"," latend_id = F.normalize(latend_id, p=2, dim=1)\n"," target_id_norm_list.append(latend_id.clone())\n"," \n"," assert len(target_id_norm_list) == len(source_specific_id_nonorm_list), \"The number of images in source and target directory must be same !!!\"\n"," video_swap(opt.video_path, target_id_norm_list,source_specific_id_nonorm_list, opt.id_thres, \\\n"," model, app, opt.output_path,temp_results_dir=opt.temp_path,no_simswaplogo=opt.no_simswaplogo,use_mask=opt.use_mask)"],"execution_count":12,"outputs":[{"output_type":"stream","text":["input mean and std: 127.5 127.5\n","find model: ./insightface_func/models/antelope/glintr100.onnx recognition\n","find model: ./insightface_func/models/antelope/scrfd_10g_bnkps.onnx detection\n","set det-size: (640, 640)\n"],"name":"stdout"},{"output_type":"stream","text":["\r 0%| | 0/594 [00:00>>> Building video ./output/multi_test_multispecific.mp4\n","[MoviePy] Writing audio in multi_test_multispecificTEMP_MPY_wvf_snd.mp3\n"],"name":"stdout"},{"output_type":"stream","text":["100%|██████████| 438/438 [00:00<00:00, 832.16it/s]"],"name":"stderr"},{"output_type":"stream","text":["[MoviePy] Done.\n","[MoviePy] Writing video ./output/multi_test_multispecific.mp4\n"],"name":"stdout"},{"output_type":"stream","text":["\n","100%|██████████| 595/595 [00:53<00:00, 11.11it/s]\n"],"name":"stderr"},{"output_type":"stream","text":["[MoviePy] Done.\n","[MoviePy] >>>> Video ready: ./output/multi_test_multispecific.mp4 \n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Rty2GsyZZrI6"},"source":[],"execution_count":null,"outputs":[]}]} \ No newline at end of file diff --git a/SimSwap colab.ipynb b/SimSwap colab.ipynb index 8af741f..5b07fa2 100644 --- a/SimSwap colab.ipynb +++ b/SimSwap colab.ipynb @@ -398,7 +398,7 @@ "opt.isTrain = False\n", "opt.use_mask = True ## new feature up-to-date\n", "\n", - "crop_size = 224\n", + "crop_size = opt.crop_size\n", "\n", "torch.nn.Module.dump_patches = True\n", "model = create_model(opt)\n", @@ -420,7 +420,7 @@ " img_id = img_id.cuda()\n", "\n", " #create latent id\n", - " img_id_downsample = F.interpolate(img_id, scale_factor=0.5)\n", + " img_id_downsample = F.interpolate(img_id, size=(112,112))\n", " latend_id = model.netArc(img_id_downsample)\n", " latend_id = latend_id.detach().to('cpu')\n", " latend_id = latend_id/np.linalg.norm(latend_id,axis=1,keepdims=True)\n", diff --git a/insightface_func/face_detect_crop_multi.py b/insightface_func/face_detect_crop_multi.py index acbf615..d75c36b 100644 --- a/insightface_func/face_detect_crop_multi.py +++ b/insightface_func/face_detect_crop_multi.py @@ -1,3 +1,11 @@ +''' +Author: Naiyuan liu +Github: https://github.com/NNNNAI +Date: 2021-11-23 17:03:58 +LastEditors: Naiyuan liu +LastEditTime: 2021-11-24 16:45:41 +Description: +''' from __future__ import division import collections import numpy as np @@ -6,7 +14,7 @@ import os import os.path as osp import cv2 from insightface.model_zoo import model_zoo -from insightface.utils import face_align +from insightface_func.utils import face_align_ffhqandnewarc as face_align __all__ = ['Face_detect_crop', 'Face'] @@ -40,8 +48,9 @@ class Face_detect_crop: self.det_model = self.models['detection'] - def prepare(self, ctx_id, det_thresh=0.5, det_size=(640, 640)): + def prepare(self, ctx_id, det_thresh=0.5, det_size=(640, 640), mode ='None'): self.det_thresh = det_thresh + self.mode = mode assert det_size is not None print('set det-size:', det_size) self.det_size = det_size @@ -73,7 +82,7 @@ class Face_detect_crop: kps = None if kpss is not None: kps = kpss[i] - M, _ = face_align.estimate_norm(kps, crop_size, mode ='None') + M, _ = face_align.estimate_norm(kps, crop_size, mode = self.mode) align_img = cv2.warpAffine(img, M, (crop_size, crop_size), borderValue=0.0) align_img_list.append(align_img) M_list.append(M) diff --git a/insightface_func/face_detect_crop_single.py b/insightface_func/face_detect_crop_single.py index 1fdd5d9..f3f648f 100644 --- a/insightface_func/face_detect_crop_single.py +++ b/insightface_func/face_detect_crop_single.py @@ -1,3 +1,11 @@ +''' +Author: Naiyuan liu +Github: https://github.com/NNNNAI +Date: 2021-11-23 17:03:58 +LastEditors: Naiyuan liu +LastEditTime: 2021-11-24 16:46:04 +Description: +''' from __future__ import division import collections import numpy as np @@ -6,7 +14,7 @@ import os import os.path as osp import cv2 from insightface.model_zoo import model_zoo -from insightface.utils import face_align +from insightface_func.utils import face_align_ffhqandnewarc as face_align __all__ = ['Face_detect_crop', 'Face'] @@ -40,8 +48,9 @@ class Face_detect_crop: self.det_model = self.models['detection'] - def prepare(self, ctx_id, det_thresh=0.5, det_size=(640, 640)): + def prepare(self, ctx_id, det_thresh=0.5, det_size=(640, 640), mode ='None'): self.det_thresh = det_thresh + self.mode = mode assert det_size is not None print('set det-size:', det_size) self.det_size = det_size @@ -82,7 +91,7 @@ class Face_detect_crop: kps = None if kpss is not None: kps = kpss[best_index] - M, _ = face_align.estimate_norm(kps, crop_size, mode ='None') + M, _ = face_align.estimate_norm(kps, crop_size, mode = self.mode) align_img = cv2.warpAffine(img, M, (crop_size, crop_size), borderValue=0.0) return [align_img], [M] diff --git a/insightface_func/utils/face_align_ffhqandnewarc.py b/insightface_func/utils/face_align_ffhqandnewarc.py new file mode 100644 index 0000000..2f5aeb7 --- /dev/null +++ b/insightface_func/utils/face_align_ffhqandnewarc.py @@ -0,0 +1,159 @@ +''' +Author: Naiyuan liu +Github: https://github.com/NNNNAI +Date: 2021-11-15 19:42:42 +LastEditors: Naiyuan liu +LastEditTime: 2021-11-15 20:01:47 +Description: +''' + +import cv2 +import numpy as np +from skimage import transform as trans + +src1 = np.array([[51.642, 50.115], [57.617, 49.990], [35.740, 69.007], + [51.157, 89.050], [57.025, 89.702]], + dtype=np.float32) +#<--left +src2 = np.array([[45.031, 50.118], [65.568, 50.872], [39.677, 68.111], + [45.177, 86.190], [64.246, 86.758]], + dtype=np.float32) + +#---frontal +src3 = np.array([[39.730, 51.138], [72.270, 51.138], [56.000, 68.493], + [42.463, 87.010], [69.537, 87.010]], + dtype=np.float32) + +#-->right +src4 = np.array([[46.845, 50.872], [67.382, 50.118], [72.737, 68.111], + [48.167, 86.758], [67.236, 86.190]], + dtype=np.float32) + +#-->right profile +src5 = np.array([[54.796, 49.990], [60.771, 50.115], [76.673, 69.007], + [55.388, 89.702], [61.257, 89.050]], + dtype=np.float32) + +src = np.array([src1, src2, src3, src4, src5]) +src_map = src + +ffhq_src = np.array([[192.98138, 239.94708], [318.90277, 240.1936], [256.63416, 314.01935], + [201.26117, 371.41043], [313.08905, 371.15118]]) +ffhq_src = np.expand_dims(ffhq_src, axis=0) + +# arcface_src = np.array( +# [[38.2946, 51.6963], [73.5318, 51.5014], [56.0252, 71.7366], +# [41.5493, 92.3655], [70.7299, 92.2041]], +# dtype=np.float32) + +# arcface_src = np.expand_dims(arcface_src, axis=0) + +# In[66]: + + +# lmk is prediction; src is template +def estimate_norm(lmk, image_size=112, mode='ffhq'): + assert lmk.shape == (5, 2) + tform = trans.SimilarityTransform() + lmk_tran = np.insert(lmk, 2, values=np.ones(5), axis=1) + min_M = [] + min_index = [] + min_error = float('inf') + if mode == 'ffhq': + # assert image_size == 112 + src = ffhq_src * image_size / 512 + else: + src = src_map * image_size / 112 + for i in np.arange(src.shape[0]): + tform.estimate(lmk, src[i]) + M = tform.params[0:2, :] + results = np.dot(M, lmk_tran.T) + results = results.T + error = np.sum(np.sqrt(np.sum((results - src[i])**2, axis=1))) + # print(error) + if error < min_error: + min_error = error + min_M = M + min_index = i + return min_M, min_index + + +def norm_crop(img, landmark, image_size=112, mode='ffhq'): + if mode == 'Both': + M_None, _ = estimate_norm(landmark, image_size, mode = 'newarc') + M_ffhq, _ = estimate_norm(landmark, image_size, mode='ffhq') + warped_None = cv2.warpAffine(img, M_None, (image_size, image_size), borderValue=0.0) + warped_ffhq = cv2.warpAffine(img, M_ffhq, (image_size, image_size), borderValue=0.0) + return warped_ffhq, warped_None + else: + M, pose_index = estimate_norm(landmark, image_size, mode) + warped = cv2.warpAffine(img, M, (image_size, image_size), borderValue=0.0) + return warped + +def square_crop(im, S): + if im.shape[0] > im.shape[1]: + height = S + width = int(float(im.shape[1]) / im.shape[0] * S) + scale = float(S) / im.shape[0] + else: + width = S + height = int(float(im.shape[0]) / im.shape[1] * S) + scale = float(S) / im.shape[1] + resized_im = cv2.resize(im, (width, height)) + det_im = np.zeros((S, S, 3), dtype=np.uint8) + det_im[:resized_im.shape[0], :resized_im.shape[1], :] = resized_im + return det_im, scale + + +def transform(data, center, output_size, scale, rotation): + scale_ratio = scale + rot = float(rotation) * np.pi / 180.0 + #translation = (output_size/2-center[0]*scale_ratio, output_size/2-center[1]*scale_ratio) + t1 = trans.SimilarityTransform(scale=scale_ratio) + cx = center[0] * scale_ratio + cy = center[1] * scale_ratio + t2 = trans.SimilarityTransform(translation=(-1 * cx, -1 * cy)) + t3 = trans.SimilarityTransform(rotation=rot) + t4 = trans.SimilarityTransform(translation=(output_size / 2, + output_size / 2)) + t = t1 + t2 + t3 + t4 + M = t.params[0:2] + cropped = cv2.warpAffine(data, + M, (output_size, output_size), + borderValue=0.0) + return cropped, M + + +def trans_points2d(pts, M): + new_pts = np.zeros(shape=pts.shape, dtype=np.float32) + for i in range(pts.shape[0]): + pt = pts[i] + new_pt = np.array([pt[0], pt[1], 1.], dtype=np.float32) + new_pt = np.dot(M, new_pt) + #print('new_pt', new_pt.shape, new_pt) + new_pts[i] = new_pt[0:2] + + return new_pts + + +def trans_points3d(pts, M): + scale = np.sqrt(M[0][0] * M[0][0] + M[0][1] * M[0][1]) + #print(scale) + new_pts = np.zeros(shape=pts.shape, dtype=np.float32) + for i in range(pts.shape[0]): + pt = pts[i] + new_pt = np.array([pt[0], pt[1], 1.], dtype=np.float32) + new_pt = np.dot(M, new_pt) + #print('new_pt', new_pt.shape, new_pt) + new_pts[i][0:2] = new_pt[0:2] + new_pts[i][2] = pts[i][2] * scale + + return new_pts + + +def trans_points(pts, M): + if pts.shape[1] == 2: + return trans_points2d(pts, M) + else: + return trans_points3d(pts, M) + diff --git a/models/fs_model.py b/models/fs_model.py index 49cb066..548dd92 100644 --- a/models/fs_model.py +++ b/models/fs_model.py @@ -4,10 +4,8 @@ import torch.nn as nn import torch.nn.functional as F import os from torch.autograd import Variable -from util.image_pool import ImagePool from .base_model import BaseModel from . import networks -from .fs_networks import Generator_Adain_Upsample, Discriminator class SpecificNorm(nn.Module): def __init__(self, epsilon=1e-8): @@ -52,6 +50,11 @@ class fsModel(BaseModel): device = torch.device("cuda:0") + if opt.crop_size == 224: + from .fs_networks import Generator_Adain_Upsample, Discriminator + elif opt.crop_size == 512: + from .fs_networks_512 import Generator_Adain_Upsample, Discriminator + # Generator network self.netG = Generator_Adain_Upsample(input_nc=3, output_nc=3, latent_size=512, n_blocks=9, deep=False) self.netG.to(device) @@ -197,7 +200,7 @@ class fsModel(BaseModel): #G_ID - img_fake_down = F.interpolate(img_fake, scale_factor=0.5) + img_fake_down = F.interpolate(img_fake, size=(112,112)) img_fake_down = self.spNorm(img_fake_down) latent_fake = self.netArc(img_fake_down) loss_G_ID = (1 - self.cosin_metric(latent_fake, latent_id)) diff --git a/models/fs_networks_512.py b/models/fs_networks_512.py new file mode 100644 index 0000000..1b9f9c8 --- /dev/null +++ b/models/fs_networks_512.py @@ -0,0 +1,232 @@ +''' +Author: Naiyuan liu +Github: https://github.com/NNNNAI +Date: 2021-11-23 16:55:48 +LastEditors: Naiyuan liu +LastEditTime: 2021-11-24 16:58:06 +Description: +''' +""" +Copyright (C) 2019 NVIDIA Corporation. All rights reserved. +Licensed under the CC BY-NC-SA 4.0 license (https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode). +""" + +import torch +import torch.nn as nn + + +class InstanceNorm(nn.Module): + def __init__(self, epsilon=1e-8): + """ + @notice: avoid in-place ops. + https://discuss.pytorch.org/t/encounter-the-runtimeerror-one-of-the-variables-needed-for-gradient-computation-has-been-modified-by-an-inplace-operation/836/3 + """ + super(InstanceNorm, self).__init__() + self.epsilon = epsilon + + def forward(self, x): + x = x - torch.mean(x, (2, 3), True) + tmp = torch.mul(x, x) # or x ** 2 + tmp = torch.rsqrt(torch.mean(tmp, (2, 3), True) + self.epsilon) + return x * tmp + +class ApplyStyle(nn.Module): + """ + @ref: https://github.com/lernapparat/lernapparat/blob/master/style_gan/pytorch_style_gan.ipynb + """ + def __init__(self, latent_size, channels): + super(ApplyStyle, self).__init__() + self.linear = nn.Linear(latent_size, channels * 2) + + def forward(self, x, latent): + style = self.linear(latent) # style => [batch_size, n_channels*2] + shape = [-1, 2, x.size(1), 1, 1] + style = style.view(shape) # [batch_size, 2, n_channels, ...] + #x = x * (style[:, 0] + 1.) + style[:, 1] + x = x * (style[:, 0] * 1 + 1.) + style[:, 1] * 1 + return x + +class ResnetBlock_Adain(nn.Module): + def __init__(self, dim, latent_size, padding_type, activation=nn.ReLU(True)): + super(ResnetBlock_Adain, self).__init__() + + p = 0 + conv1 = [] + if padding_type == 'reflect': + conv1 += [nn.ReflectionPad2d(1)] + elif padding_type == 'replicate': + conv1 += [nn.ReplicationPad2d(1)] + elif padding_type == 'zero': + p = 1 + else: + raise NotImplementedError('padding [%s] is not implemented' % padding_type) + conv1 += [nn.Conv2d(dim, dim, kernel_size=3, padding = p), InstanceNorm()] + self.conv1 = nn.Sequential(*conv1) + self.style1 = ApplyStyle(latent_size, dim) + self.act1 = activation + + p = 0 + conv2 = [] + if padding_type == 'reflect': + conv2 += [nn.ReflectionPad2d(1)] + elif padding_type == 'replicate': + conv2 += [nn.ReplicationPad2d(1)] + elif padding_type == 'zero': + p = 1 + else: + raise NotImplementedError('padding [%s] is not implemented' % padding_type) + conv2 += [nn.Conv2d(dim, dim, kernel_size=3, padding=p), InstanceNorm()] + self.conv2 = nn.Sequential(*conv2) + self.style2 = ApplyStyle(latent_size, dim) + + + def forward(self, x, dlatents_in_slice): + y = self.conv1(x) + y = self.style1(y, dlatents_in_slice) + y = self.act1(y) + y = self.conv2(y) + y = self.style2(y, dlatents_in_slice) + out = x + y + return out + + + +class Generator_Adain_Upsample(nn.Module): + def __init__(self, input_nc, output_nc, latent_size, n_blocks=6, deep=False, + norm_layer=nn.BatchNorm2d, + padding_type='reflect'): + assert (n_blocks >= 0) + super(Generator_Adain_Upsample, self).__init__() + activation = nn.ReLU(True) + self.deep = deep + + self.first_layer = nn.Sequential(nn.ReflectionPad2d(3), nn.Conv2d(input_nc, 32, kernel_size=7, padding=0), + norm_layer(32), activation) + ### downsample + self.down0 = nn.Sequential(nn.Conv2d(32, 64, kernel_size=3, stride=2, padding=1), + norm_layer(64), activation) + self.down1 = nn.Sequential(nn.Conv2d(64, 128, kernel_size=3, stride=2, padding=1), + norm_layer(128), activation) + self.down2 = nn.Sequential(nn.Conv2d(128, 256, kernel_size=3, stride=2, padding=1), + norm_layer(256), activation) + self.down3 = nn.Sequential(nn.Conv2d(256, 512, kernel_size=3, stride=2, padding=1), + norm_layer(512), activation) + if self.deep: + self.down4 = nn.Sequential(nn.Conv2d(512, 512, kernel_size=3, stride=2, padding=1), + norm_layer(512), activation) + + ### resnet blocks + BN = [] + for i in range(n_blocks): + BN += [ + ResnetBlock_Adain(512, latent_size=latent_size, padding_type=padding_type, activation=activation)] + self.BottleNeck = nn.Sequential(*BN) + + if self.deep: + self.up4 = nn.Sequential( + nn.Upsample(scale_factor=2, mode='bilinear'), + nn.Conv2d(512, 512, kernel_size=3, stride=1, padding=1), + nn.BatchNorm2d(512), activation + ) + self.up3 = nn.Sequential( + nn.Upsample(scale_factor=2, mode='bilinear'), + nn.Conv2d(512, 256, kernel_size=3, stride=1, padding=1), + nn.BatchNorm2d(256), activation + ) + self.up2 = nn.Sequential( + nn.Upsample(scale_factor=2, mode='bilinear'), + nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=1), + nn.BatchNorm2d(128), activation + ) + self.up1 = nn.Sequential( + nn.Upsample(scale_factor=2, mode='bilinear'), + nn.Conv2d(128, 64, kernel_size=3, stride=1, padding=1), + nn.BatchNorm2d(64), activation + ) + self.up0 = nn.Sequential( + nn.Upsample(scale_factor=2, mode='bilinear'), + nn.Conv2d(64, 32, kernel_size=3, stride=1, padding=1), + nn.BatchNorm2d(32), activation + ) + self.last_layer = nn.Sequential(nn.ReflectionPad2d(3), nn.Conv2d(32, output_nc, kernel_size=7, padding=0), + nn.Tanh()) + + def forward(self, input, dlatents): + x = input # 3*224*224 + + skip0 = self.first_layer(x) + skip1 = self.down0(skip0) + skip2 = self.down1(skip1) + skip3 = self.down2(skip2) + if self.deep: + skip4 = self.down3(skip3) + x = self.down4(skip4) + else: + x = self.down3(skip3) + + for i in range(len(self.BottleNeck)): + x = self.BottleNeck[i](x, dlatents) + + if self.deep: + x = self.up4(x) + x = self.up3(x) + x = self.up2(x) + x = self.up1(x) + x = self.up0(x) + x = self.last_layer(x) + x = (x + 1) / 2 + + return x + +class Discriminator(nn.Module): + def __init__(self, input_nc, norm_layer=nn.BatchNorm2d, use_sigmoid=False): + super(Discriminator, self).__init__() + + kw = 4 + padw = 1 + self.down1 = nn.Sequential( + nn.Conv2d(input_nc, 64, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True) + ) + self.down2 = nn.Sequential( + nn.Conv2d(64, 128, kernel_size=kw, stride=2, padding=padw), + norm_layer(128), nn.LeakyReLU(0.2, True) + ) + self.down3 = nn.Sequential( + nn.Conv2d(128, 256, kernel_size=kw, stride=2, padding=padw), + norm_layer(256), nn.LeakyReLU(0.2, True) + ) + self.down4 = nn.Sequential( + nn.Conv2d(256, 512, kernel_size=kw, stride=2, padding=padw), + norm_layer(512), nn.LeakyReLU(0.2, True) + ) + self.conv1 = nn.Sequential( + nn.Conv2d(512, 512, kernel_size=kw, stride=1, padding=padw), + norm_layer(512), + nn.LeakyReLU(0.2, True) + ) + + if use_sigmoid: + self.conv2 = nn.Sequential( + nn.Conv2d(512, 1, kernel_size=kw, stride=1, padding=padw), nn.Sigmoid() + ) + else: + self.conv2 = nn.Sequential( + nn.Conv2d(512, 1, kernel_size=kw, stride=1, padding=padw) + ) + + def forward(self, input): + out = [] + x = self.down1(input) + out.append(x) + x = self.down2(x) + out.append(x) + x = self.down3(x) + out.append(x) + x = self.down4(x) + out.append(x) + x = self.conv1(x) + out.append(x) + x = self.conv2(x) + out.append(x) + + return out diff --git a/options/test_options.py b/options/test_options.py index 0262186..18c5527 100644 --- a/options/test_options.py +++ b/options/test_options.py @@ -1,3 +1,11 @@ +''' +Author: Naiyuan liu +Github: https://github.com/NNNNAI +Date: 2021-11-23 17:03:58 +LastEditors: Naiyuan liu +LastEditTime: 2021-11-23 17:08:08 +Description: +''' from .base_options import BaseOptions class TestOptions(BaseOptions): @@ -25,6 +33,6 @@ class TestOptions(BaseOptions): self.parser.add_argument('--id_thres', type=float, default=0.03, help='how many test images to run') self.parser.add_argument('--no_simswaplogo', action='store_true', help='Remove the watermark') self.parser.add_argument('--use_mask', action='store_true', help='Use mask for better result') + self.parser.add_argument('--crop_size', type=int, default=224, help='Crop of size of input image') - self.isTrain = False diff --git a/predict.py b/predict.py index 8f930dd..3fe60e3 100644 --- a/predict.py +++ b/predict.py @@ -56,7 +56,7 @@ class Predictor(cog.Predictor): model = create_model(opt) model.eval() - crop_size = 224 + crop_size = opt.crop_size spNorm = SpecificNorm() with torch.no_grad(): @@ -71,7 +71,7 @@ class Predictor(cog.Predictor): img_id = img_id.cuda() # create latent id - img_id_downsample = F.interpolate(img_id, scale_factor=0.5) + img_id_downsample = F.interpolate(img_id, size=(112,112)) latend_id = model.netArc(img_id_downsample) latend_id = F.normalize(latend_id, p=2, dim=1) diff --git a/test_one_image.py b/test_one_image.py index a05032b..4eabd8e 100644 --- a/test_one_image.py +++ b/test_one_image.py @@ -53,7 +53,7 @@ if __name__ == '__main__': img_att = img_att.cuda() #create latent id - img_id_downsample = F.interpolate(img_id, scale_factor=0.5) + img_id_downsample = F.interpolate(img_id, size=(112,112)) latend_id = model.netArc(img_id_downsample) latend_id = latend_id.detach().to('cpu') latend_id = latend_id/np.linalg.norm(latend_id,axis=1,keepdims=True) diff --git a/test_video_swap_multispecific.py b/test_video_swap_multispecific.py index 0db7592..0143157 100644 --- a/test_video_swap_multispecific.py +++ b/test_video_swap_multispecific.py @@ -35,16 +35,22 @@ if __name__ == '__main__': opt = TestOptions().parse() pic_specific = opt.pic_specific_path start_epoch, epoch_iter = 1, 0 - crop_size = 224 + crop_size = opt.crop_size multisepcific_dir = opt.multisepcific_dir torch.nn.Module.dump_patches = True + if crop_size == 512: + opt.which_epoch = 550000 + opt.name = '512' + mode = 'ffhq' + else: + mode = 'None' model = create_model(opt) model.eval() app = Face_detect_crop(name='antelope', root='./insightface_func/models') - app.prepare(ctx_id= 0, det_thresh=0.6, det_size=(640,640)) + app.prepare(ctx_id= 0, det_thresh=0.6, det_size=(640,640),mode=mode) # The specific person to be swapped(source) @@ -61,7 +67,7 @@ if __name__ == '__main__': # convert numpy to tensor specific_person = specific_person.cuda() #create latent id - specific_person_downsample = F.interpolate(specific_person, scale_factor=0.5) + specific_person_downsample = F.interpolate(specific_person, size=(112,112)) specific_person_id_nonorm = model.netArc(specific_person_downsample) source_specific_id_nonorm_list.append(specific_person_id_nonorm.clone()) @@ -80,7 +86,7 @@ if __name__ == '__main__': # convert numpy to tensor img_id = img_id.cuda() #create latent id - img_id_downsample = F.interpolate(img_id, scale_factor=0.5) + img_id_downsample = F.interpolate(img_id, size=(112,112)) latend_id = model.netArc(img_id_downsample) latend_id = F.normalize(latend_id, p=2, dim=1) target_id_norm_list.append(latend_id.clone()) @@ -90,5 +96,5 @@ if __name__ == '__main__': video_swap(opt.video_path, target_id_norm_list,source_specific_id_nonorm_list, opt.id_thres, \ - model, app, opt.output_path,temp_results_dir=opt.temp_path,no_simswaplogo=opt.no_simswaplogo,use_mask=opt.use_mask) + model, app, opt.output_path,temp_results_dir=opt.temp_path,no_simswaplogo=opt.no_simswaplogo,use_mask=opt.use_mask,crop_size=crop_size) diff --git a/test_video_swapmulti.py b/test_video_swapmulti.py index c2115ea..37d7751 100644 --- a/test_video_swapmulti.py +++ b/test_video_swapmulti.py @@ -1,3 +1,11 @@ +''' +Author: Naiyuan liu +Github: https://github.com/NNNNAI +Date: 2021-11-23 17:03:58 +LastEditors: Naiyuan liu +LastEditTime: 2021-11-24 19:00:34 +Description: +''' import cv2 import torch @@ -34,15 +42,21 @@ if __name__ == '__main__': opt = TestOptions().parse() start_epoch, epoch_iter = 1, 0 - crop_size = 224 + crop_size = opt.crop_size torch.nn.Module.dump_patches = True + + if crop_size == 512: + opt.which_epoch = 550000 + opt.name = '512' + mode = 'ffhq' + else: + mode = 'None' model = create_model(opt) model.eval() - app = Face_detect_crop(name='antelope', root='./insightface_func/models') - app.prepare(ctx_id= 0, det_thresh=0.6, det_size=(640,640)) + app.prepare(ctx_id= 0, det_thresh=0.6, det_size=(640,640),mode = mode) with torch.no_grad(): pic_a = opt.pic_a_path @@ -65,10 +79,10 @@ if __name__ == '__main__': # img_att = img_att.cuda() #create latent id - img_id_downsample = F.interpolate(img_id, scale_factor=0.5) + img_id_downsample = F.interpolate(img_id, size=(112,112)) latend_id = model.netArc(img_id_downsample) latend_id = F.normalize(latend_id, p=2, dim=1) video_swap(opt.video_path, latend_id, model, app, opt.output_path,temp_results_dir=opt.temp_path,\ - no_simswaplogo=opt.no_simswaplogo,use_mask=opt.use_mask) + no_simswaplogo=opt.no_simswaplogo,use_mask=opt.use_mask,crop_size=crop_size) diff --git a/test_video_swapsingle.py b/test_video_swapsingle.py index 29fcc16..221d0a3 100644 --- a/test_video_swapsingle.py +++ b/test_video_swapsingle.py @@ -1,3 +1,11 @@ +''' +Author: Naiyuan liu +Github: https://github.com/NNNNAI +Date: 2021-11-23 17:03:58 +LastEditors: Naiyuan liu +LastEditTime: 2021-11-24 19:00:38 +Description: +''' import cv2 import torch @@ -34,15 +42,21 @@ if __name__ == '__main__': opt = TestOptions().parse() start_epoch, epoch_iter = 1, 0 - crop_size = 224 + crop_size = opt.crop_size torch.nn.Module.dump_patches = True + if crop_size == 512: + opt.which_epoch = 550000 + opt.name = '512' + mode = 'ffhq' + else: + mode = 'None' model = create_model(opt) model.eval() app = Face_detect_crop(name='antelope', root='./insightface_func/models') - app.prepare(ctx_id= 0, det_thresh=0.6, det_size=(640,640)) + app.prepare(ctx_id= 0, det_thresh=0.6, det_size=(640,640),mode=mode) with torch.no_grad(): pic_a = opt.pic_a_path # img_a = Image.open(pic_a).convert('RGB') @@ -64,10 +78,10 @@ if __name__ == '__main__': # img_att = img_att.cuda() #create latent id - img_id_downsample = F.interpolate(img_id, scale_factor=0.5) + img_id_downsample = F.interpolate(img_id, size=(112,112)) latend_id = model.netArc(img_id_downsample) latend_id = F.normalize(latend_id, p=2, dim=1) video_swap(opt.video_path, latend_id, model, app, opt.output_path,temp_results_dir=opt.temp_path,\ - no_simswaplogo=opt.no_simswaplogo,use_mask=opt.use_mask) + no_simswaplogo=opt.no_simswaplogo,use_mask=opt.use_mask,crop_size=crop_size) diff --git a/test_video_swapspecific.py b/test_video_swapspecific.py index a5a7aad..3af643d 100644 --- a/test_video_swapspecific.py +++ b/test_video_swapspecific.py @@ -1,3 +1,11 @@ +''' +Author: Naiyuan liu +Github: https://github.com/NNNNAI +Date: 2021-11-23 17:03:58 +LastEditors: Naiyuan liu +LastEditTime: 2021-11-24 19:00:42 +Description: +''' import cv2 import torch @@ -34,15 +42,21 @@ if __name__ == '__main__': opt = TestOptions().parse() pic_specific = opt.pic_specific_path start_epoch, epoch_iter = 1, 0 - crop_size = 224 + crop_size = opt.crop_size torch.nn.Module.dump_patches = True + if crop_size == 512: + opt.which_epoch = 550000 + opt.name = '512' + mode = 'ffhq' + else: + mode = 'None' model = create_model(opt) model.eval() app = Face_detect_crop(name='antelope', root='./insightface_func/models') - app.prepare(ctx_id= 0, det_thresh=0.6, det_size=(640,640)) + app.prepare(ctx_id= 0, det_thresh=0.6, det_size=(640,640),mode=mode) with torch.no_grad(): pic_a = opt.pic_a_path # img_a = Image.open(pic_a).convert('RGB') @@ -64,7 +78,7 @@ if __name__ == '__main__': # img_att = img_att.cuda() #create latent id - img_id_downsample = F.interpolate(img_id, scale_factor=0.5) + img_id_downsample = F.interpolate(img_id, size=(112,112)) latend_id = model.netArc(img_id_downsample) latend_id = F.normalize(latend_id, p=2, dim=1) @@ -76,9 +90,9 @@ if __name__ == '__main__': specific_person = transformer_Arcface(specific_person_align_crop_pil) specific_person = specific_person.view(-1, specific_person.shape[0], specific_person.shape[1], specific_person.shape[2]) specific_person = specific_person.cuda() - specific_person_downsample = F.interpolate(specific_person, scale_factor=0.5) + specific_person_downsample = F.interpolate(specific_person, size=(112,112)) specific_person_id_nonorm = model.netArc(specific_person_downsample) video_swap(opt.video_path, latend_id,specific_person_id_nonorm, opt.id_thres, \ - model, app, opt.output_path,temp_results_dir=opt.temp_path,no_simswaplogo=opt.no_simswaplogo,use_mask=opt.use_mask) + model, app, opt.output_path,temp_results_dir=opt.temp_path,no_simswaplogo=opt.no_simswaplogo,use_mask=opt.use_mask,crop_size=crop_size) diff --git a/test_wholeimage_swap_multispecific.py b/test_wholeimage_swap_multispecific.py index 326e65b..40d4d0d 100644 --- a/test_wholeimage_swap_multispecific.py +++ b/test_wholeimage_swap_multispecific.py @@ -38,11 +38,19 @@ if __name__ == '__main__': opt = TestOptions().parse() start_epoch, epoch_iter = 1, 0 - crop_size = 224 + crop_size = opt.crop_size multisepcific_dir = opt.multisepcific_dir torch.nn.Module.dump_patches = True + + if crop_size == 512: + opt.which_epoch = 550000 + opt.name = '512' + mode = 'ffhq' + else: + mode = 'None' + logoclass = watermark_image('./simswaplogo/simswaplogo.png') model = create_model(opt) model.eval() @@ -52,7 +60,7 @@ if __name__ == '__main__': app = Face_detect_crop(name='antelope', root='./insightface_func/models') - app.prepare(ctx_id= 0, det_thresh=0.6, det_size=(640,640)) + app.prepare(ctx_id= 0, det_thresh=0.6, det_size=(640,640),mode = mode) with torch.no_grad(): # The specific person to be swapped(source) @@ -70,7 +78,7 @@ if __name__ == '__main__': # convert numpy to tensor specific_person = specific_person.cuda() #create latent id - specific_person_downsample = F.interpolate(specific_person, scale_factor=0.5) + specific_person_downsample = F.interpolate(specific_person, size=(112,112)) specific_person_id_nonorm = model.netArc(specific_person_downsample) source_specific_id_nonorm_list.append(specific_person_id_nonorm.clone()) @@ -89,7 +97,7 @@ if __name__ == '__main__': # convert numpy to tensor img_id = img_id.cuda() #create latent id - img_id_downsample = F.interpolate(img_id, scale_factor=0.5) + img_id_downsample = F.interpolate(img_id, size=(112,112)) latend_id = model.netArc(img_id_downsample) latend_id = F.normalize(latend_id, p=2, dim=1) target_id_norm_list.append(latend_id.clone()) @@ -112,7 +120,7 @@ if __name__ == '__main__': b_align_crop_tenor = _totensor(cv2.cvtColor(b_align_crop,cv2.COLOR_BGR2RGB))[None,...].cuda() b_align_crop_tenor_arcnorm = spNorm(b_align_crop_tenor) - b_align_crop_tenor_arcnorm_downsample = F.interpolate(b_align_crop_tenor_arcnorm, scale_factor=0.5) + b_align_crop_tenor_arcnorm_downsample = F.interpolate(b_align_crop_tenor_arcnorm, size=(112,112)) b_align_crop_id_nonorm = model.netArc(b_align_crop_tenor_arcnorm_downsample) id_compare_values.append([]) diff --git a/test_wholeimage_swapmulti.py b/test_wholeimage_swapmulti.py index 1dd6efa..a27b57f 100644 --- a/test_wholeimage_swapmulti.py +++ b/test_wholeimage_swapmulti.py @@ -31,16 +31,22 @@ if __name__ == '__main__': opt = TestOptions().parse() start_epoch, epoch_iter = 1, 0 - crop_size = 224 + crop_size = opt.crop_size torch.nn.Module.dump_patches = True + if crop_size == 512: + opt.which_epoch = 550000 + opt.name = '512' + mode = 'ffhq' + else: + mode = 'None' logoclass = watermark_image('./simswaplogo/simswaplogo.png') model = create_model(opt) model.eval() spNorm =SpecificNorm() app = Face_detect_crop(name='antelope', root='./insightface_func/models') - app.prepare(ctx_id= 0, det_thresh=0.6, det_size=(640,640)) + app.prepare(ctx_id= 0, det_thresh=0.6, det_size=(640,640),mode=mode) with torch.no_grad(): pic_a = opt.pic_a_path @@ -55,7 +61,7 @@ if __name__ == '__main__': img_id = img_id.cuda() #create latent id - img_id_downsample = F.interpolate(img_id, scale_factor=0.5) + img_id_downsample = F.interpolate(img_id, size=(112,112)) latend_id = model.netArc(img_id_downsample) latend_id = F.normalize(latend_id, p=2, dim=1) diff --git a/test_wholeimage_swapsingle.py b/test_wholeimage_swapsingle.py index 0d63e88..b4ac085 100644 --- a/test_wholeimage_swapsingle.py +++ b/test_wholeimage_swapsingle.py @@ -30,16 +30,22 @@ if __name__ == '__main__': opt = TestOptions().parse() start_epoch, epoch_iter = 1, 0 - crop_size = 224 + crop_size = opt.crop_size torch.nn.Module.dump_patches = True + if crop_size == 512: + opt.which_epoch = 550000 + opt.name = '512' + mode = 'ffhq' + else: + mode = 'None' logoclass = watermark_image('./simswaplogo/simswaplogo.png') model = create_model(opt) model.eval() spNorm =SpecificNorm() app = Face_detect_crop(name='antelope', root='./insightface_func/models') - app.prepare(ctx_id= 0, det_thresh=0.6, det_size=(640,640)) + app.prepare(ctx_id= 0, det_thresh=0.6, det_size=(640,640),mode=mode) with torch.no_grad(): pic_a = opt.pic_a_path @@ -54,7 +60,7 @@ if __name__ == '__main__': img_id = img_id.cuda() #create latent id - img_id_downsample = F.interpolate(img_id, scale_factor=0.5) + img_id_downsample = F.interpolate(img_id, size=(112,112)) latend_id = model.netArc(img_id_downsample) latend_id = F.normalize(latend_id, p=2, dim=1) diff --git a/test_wholeimage_swapspecific.py b/test_wholeimage_swapspecific.py index ce14e11..09beadf 100644 --- a/test_wholeimage_swapspecific.py +++ b/test_wholeimage_swapspecific.py @@ -37,9 +37,15 @@ if __name__ == '__main__': opt = TestOptions().parse() start_epoch, epoch_iter = 1, 0 - crop_size = 224 + crop_size = opt.crop_size torch.nn.Module.dump_patches = True + if crop_size == 512: + opt.which_epoch = 550000 + opt.name = '512' + mode = 'ffhq' + else: + mode = 'None' logoclass = watermark_image('./simswaplogo/simswaplogo.png') model = create_model(opt) model.eval() @@ -49,7 +55,7 @@ if __name__ == '__main__': app = Face_detect_crop(name='antelope', root='./insightface_func/models') - app.prepare(ctx_id= 0, det_thresh=0.6, det_size=(640,640)) + app.prepare(ctx_id= 0, det_thresh=0.6, det_size=(640,640),mode=mode) pic_a = opt.pic_a_path pic_specific = opt.pic_specific_path @@ -65,7 +71,7 @@ if __name__ == '__main__': img_id = img_id.cuda() #create latent id - img_id_downsample = F.interpolate(img_id, scale_factor=0.5) + img_id_downsample = F.interpolate(img_id, size=(112,112)) latend_id = model.netArc(img_id_downsample) latend_id = F.normalize(latend_id, p=2, dim=1) @@ -81,7 +87,7 @@ if __name__ == '__main__': specific_person = specific_person.cuda() #create latent id - specific_person_downsample = F.interpolate(specific_person, scale_factor=0.5) + specific_person_downsample = F.interpolate(specific_person, size=(112,112)) specific_person_id_nonorm = model.netArc(specific_person_downsample) # specific_person_id_norm = F.normalize(specific_person_id_nonorm, p=2, dim=1) @@ -101,7 +107,7 @@ if __name__ == '__main__': b_align_crop_tenor = _totensor(cv2.cvtColor(b_align_crop,cv2.COLOR_BGR2RGB))[None,...].cuda() b_align_crop_tenor_arcnorm = spNorm(b_align_crop_tenor) - b_align_crop_tenor_arcnorm_downsample = F.interpolate(b_align_crop_tenor_arcnorm, scale_factor=0.5) + b_align_crop_tenor_arcnorm_downsample = F.interpolate(b_align_crop_tenor_arcnorm, size=(112,112)) b_align_crop_id_nonorm = model.netArc(b_align_crop_tenor_arcnorm_downsample) id_compare_values.append(mse(b_align_crop_id_nonorm,specific_person_id_nonorm).detach().cpu().numpy()) diff --git a/util/reverse2original.py b/util/reverse2original.py index 0c5d29d..83f1bdb 100644 --- a/util/reverse2original.py +++ b/util/reverse2original.py @@ -110,7 +110,7 @@ def reverse2wholeimage(b_align_crop_tenor_list,swaped_imgs, mats, crop_size, ori tgt_mask = encode_segmentation_rgb(vis_parsing_anno) if tgt_mask.sum() >= 5000: # face_mask_tensor = tgt_mask[...,0] + tgt_mask[...,1] - target_mask = cv2.resize(tgt_mask, (224, 224)) + target_mask = cv2.resize(tgt_mask, (crop_size, crop_size)) # print(source_img) target_image_parsing = postprocess(swaped_img, source_img[0].cpu().detach().numpy().transpose((1, 2, 0)), target_mask,smooth_mask) diff --git a/util/videoswap.py b/util/videoswap.py index f81121d..0f0deb8 100644 --- a/util/videoswap.py +++ b/util/videoswap.py @@ -79,6 +79,7 @@ def video_swap(video_path, id_vetor, swap_model, detect_model, save_path, temp_r frame_align_crop_tenor = _totensor(cv2.cvtColor(frame_align_crop,cv2.COLOR_BGR2RGB))[None,...].cuda() swap_result = swap_model(None, frame_align_crop_tenor, id_vetor, None, True)[0] + cv2.imwrite(os.path.join(temp_results_dir, 'frame_{:0>7d}.jpg'.format(frame_index)), frame) swap_result_list.append(swap_result) frame_align_crop_tenor_list.append(frame_align_crop_tenor) diff --git a/util/videoswap_multispecific.py b/util/videoswap_multispecific.py index 20b53b6..40e9488 100644 --- a/util/videoswap_multispecific.py +++ b/util/videoswap_multispecific.py @@ -83,7 +83,7 @@ def video_swap(video_path, target_id_norm_list,source_specific_id_nonorm_list,id frame_align_crop_tenor = _totensor(cv2.cvtColor(frame_align_crop,cv2.COLOR_BGR2RGB))[None,...].cuda() frame_align_crop_tenor_arcnorm = spNorm(frame_align_crop_tenor) - frame_align_crop_tenor_arcnorm_downsample = F.interpolate(frame_align_crop_tenor_arcnorm, scale_factor=0.5) + frame_align_crop_tenor_arcnorm_downsample = F.interpolate(frame_align_crop_tenor_arcnorm, size=(112,112)) frame_align_crop_crop_id_nonorm = swap_model.netArc(frame_align_crop_tenor_arcnorm_downsample) id_compare_values.append([]) for source_specific_id_nonorm_tmp in source_specific_id_nonorm_list: diff --git a/util/videoswap_specific.py b/util/videoswap_specific.py index b8d7ee6..8177efb 100644 --- a/util/videoswap_specific.py +++ b/util/videoswap_specific.py @@ -83,7 +83,7 @@ def video_swap(video_path, id_vetor,specific_person_id_nonorm,id_thres, swap_mod frame_align_crop_tenor = _totensor(cv2.cvtColor(frame_align_crop,cv2.COLOR_BGR2RGB))[None,...].cuda() frame_align_crop_tenor_arcnorm = spNorm(frame_align_crop_tenor) - frame_align_crop_tenor_arcnorm_downsample = F.interpolate(frame_align_crop_tenor_arcnorm, scale_factor=0.5) + frame_align_crop_tenor_arcnorm_downsample = F.interpolate(frame_align_crop_tenor_arcnorm, size=(112,112)) frame_align_crop_crop_id_nonorm = swap_model.netArc(frame_align_crop_tenor_arcnorm_downsample) id_compare_values.append(mse(frame_align_crop_crop_id_nonorm,specific_person_id_nonorm).detach().cpu().numpy())