ip_adapter_art_sdxl_demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ip_adapter_art.utils.csd_clip import CSD_CLIP\n",
    "from ip_adapter_art.utils.ip_adapter import (\n",
    "    load_ip_adapter,\n",
    ")\n",
    "import torch\n",
    "from transformers import CLIPImageProcessor\n",
    "from PIL import Image\n",
    "from diffusers.utils import make_image_grid, load_image\n",
    "from huggingface_hub import hf_hub_download\n",
    "from diffusers import StableDiffusionXLPipeline"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "csd_clip_path = hf_hub_download(\n",
    "    repo_id=\"AisingioroHao0/IP-Adapter-Art\", filename=\"csd_clip.pth\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ip_adapter_art_path = hf_hub_download(\n",
    "    repo_id=\"AisingioroHao0/IP-Adapter-Art\", filename=\"ip_adapter_art_sdxl_512.pth\"\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "csd_clip = torch.load(csd_clip_path).to(\"cuda\")\n",
    "csd_clip.requires_grad_(False)\n",
    "csd_clip = csd_clip.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipe = StableDiffusionXLPipeline.from_pretrained(\n",
    "    \"stabilityai/stable-diffusion-xl-base-1.0\",\n",
    "    variant=\"fp16\",\n",
    "    torch_dtype=torch.float16,\n",
    ").to(\"cuda\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_processor = CLIPImageProcessor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "load_ip_adapter(\n",
    "    pipe.unet,\n",
    "    ip_adapter_artist_path,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scale = {\"up\": {\"block_0\": [0.0, 1.0, 0.0]}}\n",
    "pipe.set_ip_adapter_scale(scale)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process Style Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image = load_image('https://github.com/aihao2000/IP-Adapter-Artist/blob/main/README.assets/example.jpg?raw=true')\n",
    "image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pixel_values = image_processor.preprocess(image, return_tensors=\"pt\").pixel_values\n",
    "_, __, style_embeds = csd_clip(pixel_values.to(\"cuda\", torch.float32))\n",
    "ip_adapter_image_embeds = torch.stack(\n",
    "    [torch.zeros_like(style_embeds).to(\"cuda\"), style_embeds]\n",
    ").to(\"cuda\", torch.float16)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Infer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = pipe(\n",
    "    ip_adapter_image_embeds=[ip_adapter_image_embeds],\n",
    "    prompt=\"A cat sitting on a table, top hat, best quality, masterpiece\",\n",
    "    negative_prompt=\"worst quality, low quality, low res, blurry, cropped image, jpeg artifacts, error, ugly, out of frame, deformed, poorly drawn\",\n",
    "    generator=torch.Generator(\"cuda\").manual_seed(42),\n",
    "    num_inference_steps=30,\n",
    "    guidance_scale=5.0,\n",
    ").images[0]\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = pipe(\n",
    "    ip_adapter_image_embeds=[ip_adapter_image_embeds],\n",
    "    prompt=\"A house covered with ice and snow.\",\n",
    "    negativ_prompt=\"multi view, worst quality, low quality, low res, blurry, cropped image, jpeg artifacts, error, ugly, out of frame, deformed, poorly drawn\",\n",
    "    generator=torch.Generator(\"cuda\").manual_seed(42),\n",
    "    num_inference_steps=30,\n",
    "    guidance_scale=5.0,\n",
    ").images[0]\n",
    "result"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}