feat: add evaluation

* CLIP score
huggingface · thliang01 · Apr 25, 2024 · Apr 25, 2024 · Apr 25, 2024 · Apr 25, 2024
commit 476e139c206380730a53c86c72ec7d8ee6a9a76b
diff --git a/notebooks/en/quantization_stable_diffusion.ipynb b/notebooks/en/quantization_stable_diffusion.ipynb
@@ -26,7 +26,7 @@
    "outputs": [],
    "source": [
     "! pip install --upgrade diffusers accelerate transformers safetensors\n",
-    "! pip install -q numpy Pillow"
+    "! pip install -q numpy Pillow torchmetrics"
    ]
   },
   {
@@ -113,9 +113,65 @@
    "outputs": [],
    "source": [
     "prompt = \"a photo of an astronaut riding a horse on mars\"\n",
-    "image = pipe(prompt).images[0]\n",
-    "image"
+    "images = pipe(prompt).images[0]\n",
+    "images"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Quantitative Evaluation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompts = [\n",
+    "    \"a photo of an astronaut riding a horse on mars\",\n",
+    "    \"A high tech solarpunk utopia in the Amazon rainforest\",\n",
+    "    # \"A pikachu fine dining with a view to the Eiffel Tower\",\n",
+    "    # \"A mecha robot in a favela in expressionist style\",\n",
+    "    # \"an insect robot preparing a delicious meal\",\n",
+    "    # \"A small cabin on top of a snowy mountain in the style of Disney, artstation\",\n",
+    "]\n",
+    "\n",
+    "images = pipe(prompts, num_images_per_prompt=1, output_type=\"np\").images\n",
+    "\n",
+    "print(images.shape)\n",
+    "# (6, 512, 512, 3)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from torchmetrics.functional.multimodal import clip_score\n",
+        "from functools import partial\n",
+    "\n",
+    "clip_score_fn = partial(clip_score, model_name_or_path=\"openai/clip-vit-base-patch16\")\n",
+    "\n",
+    "def calculate_clip_score(images, prompts):\n",
+    "    images_int = (images * 255).astype(\"uint8\")\n",
+    "    clip_score = clip_score_fn(torch.from_numpy(images_int).permute(0, 3, 1, 2), prompts).detach()\n",
+    "    return round(float(clip_score), 4)\n",
+    "\n",
+    "sd_clip_score = calculate_clip_score(images, prompts)\n",
+    "print(f\"CLIP score: {sd_clip_score}\")\n",
+    "# CLIP score: 35.7038"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {