notebooks/inference_multitask.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "import os\n",
    "import sys\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "import albumentations as A\n",
    "from albumentations.pytorch import ToTensorV2\n",
    "import cv2\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.insert(\n",
    "    0, os.path.abspath(os.path.join(os.getcwd(), \"../../DermSynth3D_private\"))\n",
    ")\n",
    "from dermsynth3d.models.model import SkinDeepLabV3\n",
    "from dermsynth3d.datasets.datasets import (\n",
    "    Fitz17KAnnotations,\n",
    "    ImageDataset,\n",
    "    Ph2Dataset,\n",
    ")\n",
    "from dermsynth3d.utils.colorconstancy import shade_of_gray_cc\n",
    "from scripts.inference import inference_multitask\n",
    "from dermsynth3d.losses.metrics import (\n",
    "    compute_results,\n",
    "    conf_mat_cells,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multitask_model = SkinDeepLabV3(multi_head=False, freeze_backbone=False)\n",
    "multitask_model.load_state_dict(torch.load(\"/path/to/trained/model\"), strict=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "multitask_model = multitask_model.to(device)\n",
    "multitask_model.eval();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "img_size = (320, 320)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assumes the model was pretrained using these values.\n",
    "preprocess_input = A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225))\n",
    "img_preprocess = A.Compose(\n",
    "    [\n",
    "        preprocess_input,\n",
    "    ]\n",
    ")\n",
    "\n",
    "# To force a resize of the input image.\n",
    "resize_func = A.Resize(\n",
    "    height=img_size[0], width=img_size[1], interpolation=cv2.INTER_CUBIC\n",
    ")\n",
    "# Perform spatial augmentation on both the image and mask.\n",
    "spatial_augment = A.Compose(\n",
    "    [\n",
    "        resize_func,\n",
    "    ]\n",
    ")\n",
    "resize_aspect_smallest = A.augmentations.geometric.resize.SmallestMaxSize(\n",
    "    max_size=img_size[0], always_apply=True\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Fitzpatrick 17k test annotations</h2>\n",
    "\n",
    "https://github.com/mattgroh/fitzpatrick17k\n",
    "\n",
    "See instructions to download the images.\n",
    "\n",
    "Update `dir_fitz_images` to the directory containing the fitzpatrick17k images.\n",
    "\n",
    "Update `dir_fitz_test_annotations` to the subset of masks provided."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Where the fitzpatrick images are stored.\n",
    "dir_fitz_images = \"../data/fitzpatrick17k/data/finalfitz17k/\"\n",
    "# Where the manual annotations are stored.\n",
    "dir_fitz_test_annotations = \"../data/fitzpatrick17k/annotations/annotations/\"\n",
    "\n",
    "fitz_test_ds = Fitz17KAnnotations(\n",
    "    dir_images=dir_fitz_images,\n",
    "    dir_targets=dir_fitz_test_annotations,\n",
    "    image_extension=\".jpg\",\n",
    "    target_extension=\".png\",\n",
    "    spatial_transform=resize_aspect_smallest,\n",
    "    image_augment=None,\n",
    "    image_preprocess=img_preprocess,\n",
    "    totensor=ToTensorV2(transpose_mask=True),\n",
    "    color_constancy=shade_of_gray_cc,\n",
    ")\n",
    "fitz_test_dataloader = DataLoader(fitz_test_ds, batch_size=1, shuffle=False)\n",
    "len(fitz_test_dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "839d91fb782f4a7b86aacc0d80cfc121",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/25 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dir_skin_preds_fitz17k_test = \"../data/fitzpatrick17k/annotations/test/skin/\"\n",
    "\n",
    "fitz_test = inference_multitask(\n",
    "    max_imgs=len(fitz_test_dataloader),\n",
    "    model=multitask_model,\n",
    "    dataloader=fitz_test_dataloader,\n",
    "    device=device,\n",
    "    save_to_disk=True,\n",
    "    return_values=True,\n",
    "    dir_anatomy_preds=\"../data/fitzpatrick17k/annotations/test/anatomy/\",\n",
    "    dir_save_images=\"../data/fitzpatrick17k/annotations/test/images/\",\n",
    "    dir_save_skin_preds=dir_skin_preds_fitz17k_test,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.61 \\pm 0.23\n",
      "0.88 \\pm 0.10\n",
      "0.60 \\pm 0.43\n"
     ]
    }
   ],
   "source": [
    "fitz_test_df = pd.DataFrame(\n",
    "    compute_results(fitz_test_ds, dir_skin_preds_fitz17k_test, pred_ext=\".png\")\n",
    ")\n",
    "\n",
    "# Skin condition.\n",
    "print(\n",
    "    \"{:.2f} \\pm {:.2f}\".format(\n",
    "        fitz_test_df.lesion_ji.mean(),\n",
    "        fitz_test_df.lesion_ji.std(),\n",
    "    )\n",
    ")\n",
    "\n",
    "# Skin.\n",
    "print(\n",
    "    \"{:.2f} \\pm {:.2f}\".format(\n",
    "        fitz_test_df.skin_ji.mean(),\n",
    "        fitz_test_df.skin_ji.std(),\n",
    "    )\n",
    ")\n",
    "\n",
    "# Nonskin.\n",
    "print(\n",
    "    \"{:.2f} \\pm {:.2f}\".format(\n",
    "        fitz_test_df.nonskin_ji.mean(),\n",
    "        fitz_test_df.nonskin_ji.std(),\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>PH2 results</h2>\n",
    "\n",
    "https://www.fc.up.pt/addi/ph2%20database.html\n",
    "\n",
    "Update `dir_images` to your path to the images."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "resize_aspect_longest = A.augmentations.geometric.resize.LongestMaxSize(\n",
    "    max_size=img_size[0],always_apply=True)\n",
    "\n",
    "ph2_img_augment = A.Compose([\n",
    "    A.GaussianBlur(blur_limit=(9,9), always_apply=True),\n",
    "])\n",
    "\n",
    "ph2_ds = Ph2Dataset(\n",
    "    dir_images='../data/ph2/images'\n",
    "    dir_targets=None,\n",
    "    name='ph2',\n",
    "    image_extension='.bmp', \n",
    "    target_extension='.bmp',\n",
    "    image_augment=ph2_img_augment,\n",
    "    spatial_transform=resize_aspect_longest,\n",
    "    image_preprocess=img_preprocess,\n",
    "    totensor=ToTensorV2(transpose_mask=True),\n",
    "    color_constancy=shade_of_gray_cc,\n",
    ")\n",
    "print(len(ph2_ds))\n",
    "ph2_dataloader = DataLoader(ph2_ds, batch_size=1, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02bad32e37344d46b31841ef2032fcef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/200 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dir_skin_preds_ph2_test = \"../data/ph2/preds/skin/\"\n",
    "\n",
    "ph2_preds = inference_multitask(\n",
    "    max_imgs=len(ph2_dataloader),\n",
    "    model=multitask_model,\n",
    "    dataloader=ph2_dataloader,\n",
    "    device=device,\n",
    "    save_to_disk=True,\n",
    "    return_values=False,\n",
    "    dir_save_images=\"../data/ph2/preds/images/\",\n",
    "    dir_save_targets=\"../data/ph2/preds/targets/\",\n",
    "    dir_save_skin_preds=dir_skin_preds_ph2_test,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.62 \\pm 0.21\n"
     ]
    }
   ],
   "source": [
    "ph2_test_df = pd.DataFrame(\n",
    "    compute_results(ph2_ds, dir_skin_preds_ph2_test, pred_ext=\".bmp\")\n",
    ")\n",
    "\n",
    "# Skin condition.\n",
    "print(\n",
    "    \"{:.2f} \\pm {:.2f}\".format(\n",
    "        ph2_test_df.lesion_ji.mean(),\n",
    "        ph2_test_df.lesion_ji.std(),\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Dermofit results</h2>\n",
    "\n",
    "https://licensing.edinburgh-innovations.ed.ac.uk/product/dermofit-image-library\n",
    "\n",
    "Requires a license and a fee.\n",
    "\n",
    "Update `dir_images` and `dir_targets` to your paths."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1300\n"
     ]
    }
   ],
   "source": [
    "dermofit_img_augment = A.Compose(\n",
    "    [\n",
    "        A.GaussianBlur(blur_limit=(3, 3), always_apply=True),\n",
    "    ]\n",
    ")\n",
    "dermofit_ds = ImageDataset(\n",
    "    dir_images=\"../data/dermofit/images/\",\n",
    "    dir_targets=\"../data/dermofit/targets/\",\n",
    "    name=\"dermofit\",\n",
    "    image_extension=\".png\",\n",
    "    target_extension=\".png\",\n",
    "    image_augment=None,\n",
    "    spatial_transform=resize_aspect_smallest,\n",
    "    image_preprocess=img_preprocess,\n",
    "    totensor=ToTensorV2(transpose_mask=True),\n",
    "    color_constancy=shade_of_gray_cc,\n",
    ")\n",
    "print(len(dermofit_ds))\n",
    "dermofit_dataloader = DataLoader(dermofit_ds, batch_size=1, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e96312da69bb4d7f80b25da250d2e0f5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1300 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dir_skin_preds_dermofit = \"../data/dermofit/predictions/\"\n",
    "\n",
    "dermofit_preds = inference_multitask(\n",
    "    max_imgs=len(dermofit_dataloader),\n",
    "    model=multitask_model,\n",
    "    dataloader=dermofit_dataloader,\n",
    "    device=device,\n",
    "    save_to_disk=True,\n",
    "    return_values=False,\n",
    "    dir_save_skin_preds=dir_skin_preds_dermofit,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.57 \\pm 0.21\n"
     ]
    }
   ],
   "source": [
    "dermofit_df = pd.DataFrame(\n",
    "    compute_results(dermofit_ds, dir_skin_preds_dermofit, pred_ext=\".png\")\n",
    ")\n",
    "\n",
    "# Skin condition.\n",
    "print(\n",
    "    \"{:.2f} \\pm {:.2f}\".format(\n",
    "        dermofit_df.lesion_ji.mean(),\n",
    "        dermofit_df.lesion_ji.std(),\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Pratheepan results on FacePhoto</h2>\n",
    "\n",
    "Data from:\n",
    "https://web.fsktm.um.edu.my/~cschan/downloads_skin_dataset.html\n",
    "\n",
    "Update `dir_images` and `dir_targets` to your data paths to the images and skin masks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "pratheepan_ds = ImageDataset(\n",
    "    dir_images=\"../data/Pratheepan_Dataset/FacePhoto/\",\n",
    "    dir_targets=\"../data/Pratheepan_Dataset/GroundT_FacePhoto/\",\n",
    "    name=\"pratheepan\",\n",
    "    image_extension=\".jpg\",\n",
    "    target_extension=\".png\",\n",
    "    image_augment=None,\n",
    "    spatial_transform=resize_aspect_smallest,\n",
    "    image_preprocess=img_preprocess,\n",
    "    totensor=ToTensorV2(transpose_mask=True),\n",
    "    color_constancy=shade_of_gray_cc,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b108e5950264bcbb4a3de34044d2d78",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/32 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dir_skin_preds_pratheepan = \"../data/Pratheepan_Dataset/predictions/skin/\"\n",
    "\n",
    "pratheepan_dataloader = DataLoader(pratheepan_ds, batch_size=1, shuffle=False)\n",
    "pratheepan_preds = inference_multitask(\n",
    "    max_imgs=len(pratheepan_ds),\n",
    "    model=multitask_model,\n",
    "    dataloader=pratheepan_dataloader,\n",
    "    device=device,\n",
    "    save_to_disk=True,\n",
    "    return_values=False,\n",
    "    dir_anatomy_preds=\"../data/Pratheepan_Dataset/predictions/anatomy/\",\n",
    "    dir_save_skin_preds=dir_skin_preds_pratheepan,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.76 \\pm 0.14\n"
     ]
    }
   ],
   "source": [
    "pratheepan_df = pd.DataFrame(\n",
    "    compute_results(pratheepan_ds, dir_skin_preds_pratheepan, pred_ext=\".png\")\n",
    ")\n",
    "\n",
    "# Skin condition.\n",
    "print(\n",
    "    \"{:.2f} \\pm {:.2f}\".format(\n",
    "        pratheepan_df.skin_ji.mean(),\n",
    "        pratheepan_df.skin_ji.std(),\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = conf_mat_cells(pratheepan_ds, dir_skin_preds_pratheepan, \".png\")\n",
    "tps = res[\"tps\"]\n",
    "fps = res[\"fps\"]\n",
    "fns = res[\"fns\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8615441543762365"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1 = (2 * np.sum(tps)) / ((2 * np.sum(tps)) + np.sum(fps) + np.sum(fns))\n",
    "f1"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "377c28a50044843c88a43bc5a026e28784430be9ba823332f2819727b62f9bca"
  },
  "kernelspec": {
   "display_name": "Python 3.8.12 ('image3d')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}