point_e/examples/text2pointcloud.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "from point_e.diffusion.configs import DIFFUSION_CONFIGS, diffusion_from_config\n",
    "from point_e.diffusion.sampler import PointCloudSampler\n",
    "from point_e.models.download import load_checkpoint\n",
    "from point_e.models.configs import MODEL_CONFIGS, model_from_config\n",
    "from point_e.util.plotting import plot_point_cloud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "print('creating base model...')\n",
    "base_name = 'base40M-textvec'\n",
    "base_model = model_from_config(MODEL_CONFIGS[base_name], device)\n",
    "base_model.eval()\n",
    "base_diffusion = diffusion_from_config(DIFFUSION_CONFIGS[base_name])\n",
    "\n",
    "print('creating upsample model...')\n",
    "upsampler_model = model_from_config(MODEL_CONFIGS['upsample'], device)\n",
    "upsampler_model.eval()\n",
    "upsampler_diffusion = diffusion_from_config(DIFFUSION_CONFIGS['upsample'])\n",
    "\n",
    "print('downloading base checkpoint...')\n",
    "base_model.load_state_dict(load_checkpoint(base_name, device))\n",
    "\n",
    "print('downloading upsampler checkpoint...')\n",
    "upsampler_model.load_state_dict(load_checkpoint('upsample', device))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampler = PointCloudSampler(\n",
    "    device=device,\n",
    "    models=[base_model, upsampler_model],\n",
    "    diffusions=[base_diffusion, upsampler_diffusion],\n",
    "    num_points=[1024, 4096 - 1024],\n",
    "    aux_channels=['R', 'G', 'B'],\n",
    "    guidance_scale=[3.0, 0.0],\n",
    "    model_kwargs_key_filter=('texts', ''), # Do not condition the upsampler at all\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set a prompt to condition on.\n",
    "prompt = 'a red motorcycle'\n",
    "\n",
    "# Produce a sample from the model.\n",
    "samples = None\n",
    "for x in tqdm(sampler.sample_batch_progressive(batch_size=1, model_kwargs=dict(texts=[prompt]))):\n",
    "    samples = x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pc = sampler.output_to_point_clouds(samples)[0]\n",
    "fig = plot_point_cloud(pc, grid_size=3, fixed_bounds=((-0.75, -0.75, -0.75),(0.75, 0.75, 0.75)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.9 64-bit ('3.9.9')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.9 (main, Aug 15 2022, 16:40:41) \n[Clang 13.1.6 (clang-1316.0.21.2.5)]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "b270b0f43bc427bcab7703c037711644cc480aac7c1cc8d2940cfaf0b447ee2e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}