We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Problems with LDSR upscaling
When using the new LDSR upscaling feature, be it via SD upscaling in img2img or directly in "Extras", black rectangles appear in the outputs on the right and at the bottom when upscaling a 230x219 photo. 

![image 7](https://user-images.githubusercontent.com/111315671/191547996-47082733-9e6a-49d0-9ba5-6df01c2bb175.png)
original

![00021](https://user-images.githubusercontent.com/111315671/191548044-7799ad8f-02d1-4ac5-9699-4957d25a9442.png)
upscaled

no issues when running it on a 512x704 image

![00151-747985091-four squirrels by Tom Whalen](https://user-images.githubusercontent.com/111315671/191548282-fd4c4572-21d9-45ee-9e76-db5060dd6469.png)
original

![00020](https://user-images.githubusercontent.com/111315671/191548167-f08ba06b-6c4f-4c5f-b3ac-2bb9662f871b.png)
upscaled

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `modules/ldsr_model_arch.py`
Content:
```
1 import gc
2 import time
3 import warnings
4 
5 import numpy as np
6 import torch
7 import torchvision
8 from PIL import Image
9 from einops import rearrange, repeat
10 from omegaconf import OmegaConf
11 
12 from ldm.models.diffusion.ddim import DDIMSampler
13 from ldm.util import instantiate_from_config, ismap
14 
15 warnings.filterwarnings("ignore", category=UserWarning)
16 
17 
18 # Create LDSR Class
19 class LDSR:
20     def load_model_from_config(self, half_attention):
21         print(f"Loading model from {self.modelPath}")
22         pl_sd = torch.load(self.modelPath, map_location="cpu")
23         sd = pl_sd["state_dict"]
24         config = OmegaConf.load(self.yamlPath)
25         model = instantiate_from_config(config.model)
26         model.load_state_dict(sd, strict=False)
27         model.cuda()
28         if half_attention:
29             model = model.half()
30 
31         model.eval()
32         return {"model": model}
33 
34     def __init__(self, model_path, yaml_path):
35         self.modelPath = model_path
36         self.yamlPath = yaml_path
37 
38     @staticmethod
39     def run(model, selected_path, custom_steps, eta):
40         example = get_cond(selected_path)
41 
42         n_runs = 1
43         guider = None
44         ckwargs = None
45         ddim_use_x0_pred = False
46         temperature = 1.
47         eta = eta
48         custom_shape = None
49 
50         height, width = example["image"].shape[1:3]
51         split_input = height >= 128 and width >= 128
52 
53         if split_input:
54             ks = 128
55             stride = 64
56             vqf = 4  #
57             model.split_input_params = {"ks": (ks, ks), "stride": (stride, stride),
58                                         "vqf": vqf,
59                                         "patch_distributed_vq": True,
60                                         "tie_braker": False,
61                                         "clip_max_weight": 0.5,
62                                         "clip_min_weight": 0.01,
63                                         "clip_max_tie_weight": 0.5,
64                                         "clip_min_tie_weight": 0.01}
65         else:
66             if hasattr(model, "split_input_params"):
67                 delattr(model, "split_input_params")
68 
69         x_t = None
70         logs = None
71         for n in range(n_runs):
72             if custom_shape is not None:
73                 x_t = torch.randn(1, custom_shape[1], custom_shape[2], custom_shape[3]).to(model.device)
74                 x_t = repeat(x_t, '1 c h w -> b c h w', b=custom_shape[0])
75 
76             logs = make_convolutional_sample(example, model,
77                                              custom_steps=custom_steps,
78                                              eta=eta, quantize_x0=False,
79                                              custom_shape=custom_shape,
80                                              temperature=temperature, noise_dropout=0.,
81                                              corrector=guider, corrector_kwargs=ckwargs, x_T=x_t,
82                                              ddim_use_x0_pred=ddim_use_x0_pred
83                                              )
84         return logs
85 
86     def super_resolution(self, image, steps=100, target_scale=2, half_attention=False):
87         model = self.load_model_from_config(half_attention)
88 
89         # Run settings
90         diffusion_steps = int(steps)
91         eta = 1.0
92 
93         down_sample_method = 'Lanczos'
94 
95         gc.collect()
96         torch.cuda.empty_cache()
97 
98         im_og = image
99         width_og, height_og = im_og.size
100         # If we can adjust the max upscale size, then the 4 below should be our variable
101         down_sample_rate = target_scale / 4
102         wd = width_og * down_sample_rate
103         hd = height_og * down_sample_rate
104         width_downsampled_pre = int(wd)
105         height_downsampled_pre = int(hd)
106 
107         if down_sample_rate != 1:
108             print(
109                 f'Downsampling from [{width_og}, {height_og}] to [{width_downsampled_pre}, {height_downsampled_pre}]')
110             im_og = im_og.resize((width_downsampled_pre, height_downsampled_pre), Image.LANCZOS)
111         else:
112             print(f"Down sample rate is 1 from {target_scale} / 4 (Not downsampling)")
113         logs = self.run(model["model"], im_og, diffusion_steps, eta)
114 
115         sample = logs["sample"]
116         sample = sample.detach().cpu()
117         sample = torch.clamp(sample, -1., 1.)
118         sample = (sample + 1.) / 2. * 255
119         sample = sample.numpy().astype(np.uint8)
120         sample = np.transpose(sample, (0, 2, 3, 1))
121         a = Image.fromarray(sample[0])
122 
123         del model
124         gc.collect()
125         torch.cuda.empty_cache()
126         return a
127 
128 
129 def get_cond(selected_path):
130     example = dict()
131     up_f = 4
132     c = selected_path.convert('RGB')
133     c = torch.unsqueeze(torchvision.transforms.ToTensor()(c), 0)
134     c_up = torchvision.transforms.functional.resize(c, size=[up_f * c.shape[2], up_f * c.shape[3]],
135                                                     antialias=True)
136     c_up = rearrange(c_up, '1 c h w -> 1 h w c')
137     c = rearrange(c, '1 c h w -> 1 h w c')
138     c = 2. * c - 1.
139 
140     c = c.to(torch.device("cuda"))
141     example["LR_image"] = c
142     example["image"] = c_up
143 
144     return example
145 
146 
147 @torch.no_grad()
148 def convsample_ddim(model, cond, steps, shape, eta=1.0, callback=None, normals_sequence=None,
149                     mask=None, x0=None, quantize_x0=False, temperature=1., score_corrector=None,
150                     corrector_kwargs=None, x_t=None
151                     ):
152     ddim = DDIMSampler(model)
153     bs = shape[0]
154     shape = shape[1:]
155     print(f"Sampling with eta = {eta}; steps: {steps}")
156     samples, intermediates = ddim.sample(steps, batch_size=bs, shape=shape, conditioning=cond, callback=callback,
157                                          normals_sequence=normals_sequence, quantize_x0=quantize_x0, eta=eta,
158                                          mask=mask, x0=x0, temperature=temperature, verbose=False,
159                                          score_corrector=score_corrector,
160                                          corrector_kwargs=corrector_kwargs, x_t=x_t)
161 
162     return samples, intermediates
163 
164 
165 @torch.no_grad()
166 def make_convolutional_sample(batch, model, custom_steps=None, eta=1.0, quantize_x0=False, custom_shape=None, temperature=1., noise_dropout=0., corrector=None,
167                               corrector_kwargs=None, x_T=None, ddim_use_x0_pred=False):
168     log = dict()
169 
170     z, c, x, xrec, xc = model.get_input(batch, model.first_stage_key,
171                                         return_first_stage_outputs=True,
172                                         force_c_encode=not (hasattr(model, 'split_input_params')
173                                                             and model.cond_stage_key == 'coordinates_bbox'),
174                                         return_original_cond=True)
175 
176     if custom_shape is not None:
177         z = torch.randn(custom_shape)
178         print(f"Generating {custom_shape[0]} samples of shape {custom_shape[1:]}")
179 
180     z0 = None
181 
182     log["input"] = x
183     log["reconstruction"] = xrec
184 
185     if ismap(xc):
186         log["original_conditioning"] = model.to_rgb(xc)
187         if hasattr(model, 'cond_stage_key'):
188             log[model.cond_stage_key] = model.to_rgb(xc)
189 
190     else:
191         log["original_conditioning"] = xc if xc is not None else torch.zeros_like(x)
192         if model.cond_stage_model:
193             log[model.cond_stage_key] = xc if xc is not None else torch.zeros_like(x)
194             if model.cond_stage_key == 'class_label':
195                 log[model.cond_stage_key] = xc[model.cond_stage_key]
196 
197     with model.ema_scope("Plotting"):
198         t0 = time.time()
199 
200         sample, intermediates = convsample_ddim(model, c, steps=custom_steps, shape=z.shape,
201                                                 eta=eta,
202                                                 quantize_x0=quantize_x0, mask=None, x0=z0,
203                                                 temperature=temperature, score_corrector=corrector, corrector_kwargs=corrector_kwargs,
204                                                 x_t=x_T)
205         t1 = time.time()
206 
207         if ddim_use_x0_pred:
208             sample = intermediates['pred_x0'][-1]
209 
210     x_sample = model.decode_first_stage(sample)
211 
212     try:
213         x_sample_noquant = model.decode_first_stage(sample, force_not_quantize=True)
214         log["sample_noquant"] = x_sample_noquant
215         log["sample_diff"] = torch.abs(x_sample_noquant - x_sample)
216     except:
217         pass
218 
219     log["sample"] = x_sample
220     log["time"] = t1 - t0
221 
222     return log
223
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/modules/ldsr_model_arch.py b/modules/ldsr_model_arch.py
--- a/modules/ldsr_model_arch.py
+++ b/modules/ldsr_model_arch.py
@@ -101,8 +101,8 @@
         down_sample_rate = target_scale / 4
         wd = width_og * down_sample_rate
         hd = height_og * down_sample_rate
-        width_downsampled_pre = int(wd)
-        height_downsampled_pre = int(hd)
+        width_downsampled_pre = int(np.ceil(wd))
+        height_downsampled_pre = int(np.ceil(hd))
 
         if down_sample_rate != 1:
             print(
@@ -110,7 +110,12 @@
             im_og = im_og.resize((width_downsampled_pre, height_downsampled_pre), Image.LANCZOS)
         else:
             print(f"Down sample rate is 1 from {target_scale} / 4 (Not downsampling)")
-        logs = self.run(model["model"], im_og, diffusion_steps, eta)
+        
+        # pad width and height to multiples of 64, pads with the edge values of image to avoid artifacts
+        pad_w, pad_h = np.max(((2, 2), np.ceil(np.array(im_og.size) / 64).astype(int)), axis=0) * 64 - im_og.size
+        im_padded = Image.fromarray(np.pad(np.array(im_og), ((0, pad_h), (0, pad_w), (0, 0)), mode='edge'))
+        
+        logs = self.run(model["model"], im_padded, diffusion_steps, eta)
 
         sample = logs["sample"]
         sample = sample.detach().cpu()
@@ -120,6 +125,9 @@
         sample = np.transpose(sample, (0, 2, 3, 1))
         a = Image.fromarray(sample[0])
 
+        # remove padding
+        a = a.crop((0, 0) + tuple(np.array(im_og.size) * 4))
+
         del model
         gc.collect()
         torch.cuda.empty_cache()

{"golden_diff": "diff --git a/modules/ldsr_model_arch.py b/modules/ldsr_model_arch.py\n--- a/modules/ldsr_model_arch.py\n+++ b/modules/ldsr_model_arch.py\n@@ -101,8 +101,8 @@\n         down_sample_rate = target_scale / 4\n         wd = width_og * down_sample_rate\n         hd = height_og * down_sample_rate\n-        width_downsampled_pre = int(wd)\n-        height_downsampled_pre = int(hd)\n+        width_downsampled_pre = int(np.ceil(wd))\n+        height_downsampled_pre = int(np.ceil(hd))\n \n         if down_sample_rate != 1:\n             print(\n@@ -110,7 +110,12 @@\n             im_og = im_og.resize((width_downsampled_pre, height_downsampled_pre), Image.LANCZOS)\n         else:\n             print(f\"Down sample rate is 1 from {target_scale} / 4 (Not downsampling)\")\n-        logs = self.run(model[\"model\"], im_og, diffusion_steps, eta)\n+        \n+        # pad width and height to multiples of 64, pads with the edge values of image to avoid artifacts\n+        pad_w, pad_h = np.max(((2, 2), np.ceil(np.array(im_og.size) / 64).astype(int)), axis=0) * 64 - im_og.size\n+        im_padded = Image.fromarray(np.pad(np.array(im_og), ((0, pad_h), (0, pad_w), (0, 0)), mode='edge'))\n+        \n+        logs = self.run(model[\"model\"], im_padded, diffusion_steps, eta)\n \n         sample = logs[\"sample\"]\n         sample = sample.detach().cpu()\n@@ -120,6 +125,9 @@\n         sample = np.transpose(sample, (0, 2, 3, 1))\n         a = Image.fromarray(sample[0])\n \n+        # remove padding\n+        a = a.crop((0, 0) + tuple(np.array(im_og.size) * 4))\n+\n         del model\n         gc.collect()\n         torch.cuda.empty_cache()\n", "issue": "Problems with LDSR upscaling\nWhen using the new LDSR upscaling feature, be it via SD upscaling in img2img or directly in \"Extras\", black rectangles appear in the outputs on the right and at the bottom when upscaling a 230x219 photo. \r\n\r\n![image 7](https://user-images.githubusercontent.com/111315671/191547996-47082733-9e6a-49d0-9ba5-6df01c2bb175.png)\r\noriginal\r\n\r\n![00021](https://user-images.githubusercontent.com/111315671/191548044-7799ad8f-02d1-4ac5-9699-4957d25a9442.png)\r\nupscaled\r\n\r\nno issues when running it on a 512x704 image\r\n\r\n![00151-747985091-four squirrels by Tom Whalen](https://user-images.githubusercontent.com/111315671/191548282-fd4c4572-21d9-45ee-9e76-db5060dd6469.png)\r\noriginal\r\n\r\n![00020](https://user-images.githubusercontent.com/111315671/191548167-f08ba06b-6c4f-4c5f-b3ac-2bb9662f871b.png)\r\nupscaled\n", "before_files": [{"content": "import gc\nimport time\nimport warnings\n\nimport numpy as np\nimport torch\nimport torchvision\nfrom PIL import Image\nfrom einops import rearrange, repeat\nfrom omegaconf import OmegaConf\n\nfrom ldm.models.diffusion.ddim import DDIMSampler\nfrom ldm.util import instantiate_from_config, ismap\n\nwarnings.filterwarnings(\"ignore\", category=UserWarning)\n\n\n# Create LDSR Class\nclass LDSR:\n    def load_model_from_config(self, half_attention):\n        print(f\"Loading model from {self.modelPath}\")\n        pl_sd = torch.load(self.modelPath, map_location=\"cpu\")\n        sd = pl_sd[\"state_dict\"]\n        config = OmegaConf.load(self.yamlPath)\n        model = instantiate_from_config(config.model)\n        model.load_state_dict(sd, strict=False)\n        model.cuda()\n        if half_attention:\n            model = model.half()\n\n        model.eval()\n        return {\"model\": model}\n\n    def __init__(self, model_path, yaml_path):\n        self.modelPath = model_path\n        self.yamlPath = yaml_path\n\n    @staticmethod\n    def run(model, selected_path, custom_steps, eta):\n        example = get_cond(selected_path)\n\n        n_runs = 1\n        guider = None\n        ckwargs = None\n        ddim_use_x0_pred = False\n        temperature = 1.\n        eta = eta\n        custom_shape = None\n\n        height, width = example[\"image\"].shape[1:3]\n        split_input = height >= 128 and width >= 128\n\n        if split_input:\n            ks = 128\n            stride = 64\n            vqf = 4  #\n            model.split_input_params = {\"ks\": (ks, ks), \"stride\": (stride, stride),\n                                        \"vqf\": vqf,\n                                        \"patch_distributed_vq\": True,\n                                        \"tie_braker\": False,\n                                        \"clip_max_weight\": 0.5,\n                                        \"clip_min_weight\": 0.01,\n                                        \"clip_max_tie_weight\": 0.5,\n                                        \"clip_min_tie_weight\": 0.01}\n        else:\n            if hasattr(model, \"split_input_params\"):\n                delattr(model, \"split_input_params\")\n\n        x_t = None\n        logs = None\n        for n in range(n_runs):\n            if custom_shape is not None:\n                x_t = torch.randn(1, custom_shape[1], custom_shape[2], custom_shape[3]).to(model.device)\n                x_t = repeat(x_t, '1 c h w -> b c h w', b=custom_shape[0])\n\n            logs = make_convolutional_sample(example, model,\n                                             custom_steps=custom_steps,\n                                             eta=eta, quantize_x0=False,\n                                             custom_shape=custom_shape,\n                                             temperature=temperature, noise_dropout=0.,\n                                             corrector=guider, corrector_kwargs=ckwargs, x_T=x_t,\n                                             ddim_use_x0_pred=ddim_use_x0_pred\n                                             )\n        return logs\n\n    def super_resolution(self, image, steps=100, target_scale=2, half_attention=False):\n        model = self.load_model_from_config(half_attention)\n\n        # Run settings\n        diffusion_steps = int(steps)\n        eta = 1.0\n\n        down_sample_method = 'Lanczos'\n\n        gc.collect()\n        torch.cuda.empty_cache()\n\n        im_og = image\n        width_og, height_og = im_og.size\n        # If we can adjust the max upscale size, then the 4 below should be our variable\n        down_sample_rate = target_scale / 4\n        wd = width_og * down_sample_rate\n        hd = height_og * down_sample_rate\n        width_downsampled_pre = int(wd)\n        height_downsampled_pre = int(hd)\n\n        if down_sample_rate != 1:\n            print(\n                f'Downsampling from [{width_og}, {height_og}] to [{width_downsampled_pre}, {height_downsampled_pre}]')\n            im_og = im_og.resize((width_downsampled_pre, height_downsampled_pre), Image.LANCZOS)\n        else:\n            print(f\"Down sample rate is 1 from {target_scale} / 4 (Not downsampling)\")\n        logs = self.run(model[\"model\"], im_og, diffusion_steps, eta)\n\n        sample = logs[\"sample\"]\n        sample = sample.detach().cpu()\n        sample = torch.clamp(sample, -1., 1.)\n        sample = (sample + 1.) / 2. * 255\n        sample = sample.numpy().astype(np.uint8)\n        sample = np.transpose(sample, (0, 2, 3, 1))\n        a = Image.fromarray(sample[0])\n\n        del model\n        gc.collect()\n        torch.cuda.empty_cache()\n        return a\n\n\ndef get_cond(selected_path):\n    example = dict()\n    up_f = 4\n    c = selected_path.convert('RGB')\n    c = torch.unsqueeze(torchvision.transforms.ToTensor()(c), 0)\n    c_up = torchvision.transforms.functional.resize(c, size=[up_f * c.shape[2], up_f * c.shape[3]],\n                                                    antialias=True)\n    c_up = rearrange(c_up, '1 c h w -> 1 h w c')\n    c = rearrange(c, '1 c h w -> 1 h w c')\n    c = 2. * c - 1.\n\n    c = c.to(torch.device(\"cuda\"))\n    example[\"LR_image\"] = c\n    example[\"image\"] = c_up\n\n    return example\n\n\n@torch.no_grad()\ndef convsample_ddim(model, cond, steps, shape, eta=1.0, callback=None, normals_sequence=None,\n                    mask=None, x0=None, quantize_x0=False, temperature=1., score_corrector=None,\n                    corrector_kwargs=None, x_t=None\n                    ):\n    ddim = DDIMSampler(model)\n    bs = shape[0]\n    shape = shape[1:]\n    print(f\"Sampling with eta = {eta}; steps: {steps}\")\n    samples, intermediates = ddim.sample(steps, batch_size=bs, shape=shape, conditioning=cond, callback=callback,\n                                         normals_sequence=normals_sequence, quantize_x0=quantize_x0, eta=eta,\n                                         mask=mask, x0=x0, temperature=temperature, verbose=False,\n                                         score_corrector=score_corrector,\n                                         corrector_kwargs=corrector_kwargs, x_t=x_t)\n\n    return samples, intermediates\n\n\n@torch.no_grad()\ndef make_convolutional_sample(batch, model, custom_steps=None, eta=1.0, quantize_x0=False, custom_shape=None, temperature=1., noise_dropout=0., corrector=None,\n                              corrector_kwargs=None, x_T=None, ddim_use_x0_pred=False):\n    log = dict()\n\n    z, c, x, xrec, xc = model.get_input(batch, model.first_stage_key,\n                                        return_first_stage_outputs=True,\n                                        force_c_encode=not (hasattr(model, 'split_input_params')\n                                                            and model.cond_stage_key == 'coordinates_bbox'),\n                                        return_original_cond=True)\n\n    if custom_shape is not None:\n        z = torch.randn(custom_shape)\n        print(f\"Generating {custom_shape[0]} samples of shape {custom_shape[1:]}\")\n\n    z0 = None\n\n    log[\"input\"] = x\n    log[\"reconstruction\"] = xrec\n\n    if ismap(xc):\n        log[\"original_conditioning\"] = model.to_rgb(xc)\n        if hasattr(model, 'cond_stage_key'):\n            log[model.cond_stage_key] = model.to_rgb(xc)\n\n    else:\n        log[\"original_conditioning\"] = xc if xc is not None else torch.zeros_like(x)\n        if model.cond_stage_model:\n            log[model.cond_stage_key] = xc if xc is not None else torch.zeros_like(x)\n            if model.cond_stage_key == 'class_label':\n                log[model.cond_stage_key] = xc[model.cond_stage_key]\n\n    with model.ema_scope(\"Plotting\"):\n        t0 = time.time()\n\n        sample, intermediates = convsample_ddim(model, c, steps=custom_steps, shape=z.shape,\n                                                eta=eta,\n                                                quantize_x0=quantize_x0, mask=None, x0=z0,\n                                                temperature=temperature, score_corrector=corrector, corrector_kwargs=corrector_kwargs,\n                                                x_t=x_T)\n        t1 = time.time()\n\n        if ddim_use_x0_pred:\n            sample = intermediates['pred_x0'][-1]\n\n    x_sample = model.decode_first_stage(sample)\n\n    try:\n        x_sample_noquant = model.decode_first_stage(sample, force_not_quantize=True)\n        log[\"sample_noquant\"] = x_sample_noquant\n        log[\"sample_diff\"] = torch.abs(x_sample_noquant - x_sample)\n    except:\n        pass\n\n    log[\"sample\"] = x_sample\n    log[\"time\"] = t1 - t0\n\n    return log\n", "path": "modules/ldsr_model_arch.py"}], "after_files": [{"content": "import gc\nimport time\nimport warnings\n\nimport numpy as np\nimport torch\nimport torchvision\nfrom PIL import Image\nfrom einops import rearrange, repeat\nfrom omegaconf import OmegaConf\n\nfrom ldm.models.diffusion.ddim import DDIMSampler\nfrom ldm.util import instantiate_from_config, ismap\n\nwarnings.filterwarnings(\"ignore\", category=UserWarning)\n\n\n# Create LDSR Class\nclass LDSR:\n    def load_model_from_config(self, half_attention):\n        print(f\"Loading model from {self.modelPath}\")\n        pl_sd = torch.load(self.modelPath, map_location=\"cpu\")\n        sd = pl_sd[\"state_dict\"]\n        config = OmegaConf.load(self.yamlPath)\n        model = instantiate_from_config(config.model)\n        model.load_state_dict(sd, strict=False)\n        model.cuda()\n        if half_attention:\n            model = model.half()\n\n        model.eval()\n        return {\"model\": model}\n\n    def __init__(self, model_path, yaml_path):\n        self.modelPath = model_path\n        self.yamlPath = yaml_path\n\n    @staticmethod\n    def run(model, selected_path, custom_steps, eta):\n        example = get_cond(selected_path)\n\n        n_runs = 1\n        guider = None\n        ckwargs = None\n        ddim_use_x0_pred = False\n        temperature = 1.\n        eta = eta\n        custom_shape = None\n\n        height, width = example[\"image\"].shape[1:3]\n        split_input = height >= 128 and width >= 128\n\n        if split_input:\n            ks = 128\n            stride = 64\n            vqf = 4  #\n            model.split_input_params = {\"ks\": (ks, ks), \"stride\": (stride, stride),\n                                        \"vqf\": vqf,\n                                        \"patch_distributed_vq\": True,\n                                        \"tie_braker\": False,\n                                        \"clip_max_weight\": 0.5,\n                                        \"clip_min_weight\": 0.01,\n                                        \"clip_max_tie_weight\": 0.5,\n                                        \"clip_min_tie_weight\": 0.01}\n        else:\n            if hasattr(model, \"split_input_params\"):\n                delattr(model, \"split_input_params\")\n\n        x_t = None\n        logs = None\n        for n in range(n_runs):\n            if custom_shape is not None:\n                x_t = torch.randn(1, custom_shape[1], custom_shape[2], custom_shape[3]).to(model.device)\n                x_t = repeat(x_t, '1 c h w -> b c h w', b=custom_shape[0])\n\n            logs = make_convolutional_sample(example, model,\n                                             custom_steps=custom_steps,\n                                             eta=eta, quantize_x0=False,\n                                             custom_shape=custom_shape,\n                                             temperature=temperature, noise_dropout=0.,\n                                             corrector=guider, corrector_kwargs=ckwargs, x_T=x_t,\n                                             ddim_use_x0_pred=ddim_use_x0_pred\n                                             )\n        return logs\n\n    def super_resolution(self, image, steps=100, target_scale=2, half_attention=False):\n        model = self.load_model_from_config(half_attention)\n\n        # Run settings\n        diffusion_steps = int(steps)\n        eta = 1.0\n\n        down_sample_method = 'Lanczos'\n\n        gc.collect()\n        torch.cuda.empty_cache()\n\n        im_og = image\n        width_og, height_og = im_og.size\n        # If we can adjust the max upscale size, then the 4 below should be our variable\n        down_sample_rate = target_scale / 4\n        wd = width_og * down_sample_rate\n        hd = height_og * down_sample_rate\n        width_downsampled_pre = int(np.ceil(wd))\n        height_downsampled_pre = int(np.ceil(hd))\n\n        if down_sample_rate != 1:\n            print(\n                f'Downsampling from [{width_og}, {height_og}] to [{width_downsampled_pre}, {height_downsampled_pre}]')\n            im_og = im_og.resize((width_downsampled_pre, height_downsampled_pre), Image.LANCZOS)\n        else:\n            print(f\"Down sample rate is 1 from {target_scale} / 4 (Not downsampling)\")\n        \n        # pad width and height to multiples of 64, pads with the edge values of image to avoid artifacts\n        pad_w, pad_h = np.max(((2, 2), np.ceil(np.array(im_og.size) / 64).astype(int)), axis=0) * 64 - im_og.size\n        im_padded = Image.fromarray(np.pad(np.array(im_og), ((0, pad_h), (0, pad_w), (0, 0)), mode='edge'))\n        \n        logs = self.run(model[\"model\"], im_padded, diffusion_steps, eta)\n\n        sample = logs[\"sample\"]\n        sample = sample.detach().cpu()\n        sample = torch.clamp(sample, -1., 1.)\n        sample = (sample + 1.) / 2. * 255\n        sample = sample.numpy().astype(np.uint8)\n        sample = np.transpose(sample, (0, 2, 3, 1))\n        a = Image.fromarray(sample[0])\n\n        # remove padding\n        a = a.crop((0, 0) + tuple(np.array(im_og.size) * 4))\n\n        del model\n        gc.collect()\n        torch.cuda.empty_cache()\n        return a\n\n\ndef get_cond(selected_path):\n    example = dict()\n    up_f = 4\n    c = selected_path.convert('RGB')\n    c = torch.unsqueeze(torchvision.transforms.ToTensor()(c), 0)\n    c_up = torchvision.transforms.functional.resize(c, size=[up_f * c.shape[2], up_f * c.shape[3]],\n                                                    antialias=True)\n    c_up = rearrange(c_up, '1 c h w -> 1 h w c')\n    c = rearrange(c, '1 c h w -> 1 h w c')\n    c = 2. * c - 1.\n\n    c = c.to(torch.device(\"cuda\"))\n    example[\"LR_image\"] = c\n    example[\"image\"] = c_up\n\n    return example\n\n\n@torch.no_grad()\ndef convsample_ddim(model, cond, steps, shape, eta=1.0, callback=None, normals_sequence=None,\n                    mask=None, x0=None, quantize_x0=False, temperature=1., score_corrector=None,\n                    corrector_kwargs=None, x_t=None\n                    ):\n    ddim = DDIMSampler(model)\n    bs = shape[0]\n    shape = shape[1:]\n    print(f\"Sampling with eta = {eta}; steps: {steps}\")\n    samples, intermediates = ddim.sample(steps, batch_size=bs, shape=shape, conditioning=cond, callback=callback,\n                                         normals_sequence=normals_sequence, quantize_x0=quantize_x0, eta=eta,\n                                         mask=mask, x0=x0, temperature=temperature, verbose=False,\n                                         score_corrector=score_corrector,\n                                         corrector_kwargs=corrector_kwargs, x_t=x_t)\n\n    return samples, intermediates\n\n\n@torch.no_grad()\ndef make_convolutional_sample(batch, model, custom_steps=None, eta=1.0, quantize_x0=False, custom_shape=None, temperature=1., noise_dropout=0., corrector=None,\n                              corrector_kwargs=None, x_T=None, ddim_use_x0_pred=False):\n    log = dict()\n\n    z, c, x, xrec, xc = model.get_input(batch, model.first_stage_key,\n                                        return_first_stage_outputs=True,\n                                        force_c_encode=not (hasattr(model, 'split_input_params')\n                                                            and model.cond_stage_key == 'coordinates_bbox'),\n                                        return_original_cond=True)\n\n    if custom_shape is not None:\n        z = torch.randn(custom_shape)\n        print(f\"Generating {custom_shape[0]} samples of shape {custom_shape[1:]}\")\n\n    z0 = None\n\n    log[\"input\"] = x\n    log[\"reconstruction\"] = xrec\n\n    if ismap(xc):\n        log[\"original_conditioning\"] = model.to_rgb(xc)\n        if hasattr(model, 'cond_stage_key'):\n            log[model.cond_stage_key] = model.to_rgb(xc)\n\n    else:\n        log[\"original_conditioning\"] = xc if xc is not None else torch.zeros_like(x)\n        if model.cond_stage_model:\n            log[model.cond_stage_key] = xc if xc is not None else torch.zeros_like(x)\n            if model.cond_stage_key == 'class_label':\n                log[model.cond_stage_key] = xc[model.cond_stage_key]\n\n    with model.ema_scope(\"Plotting\"):\n        t0 = time.time()\n\n        sample, intermediates = convsample_ddim(model, c, steps=custom_steps, shape=z.shape,\n                                                eta=eta,\n                                                quantize_x0=quantize_x0, mask=None, x0=z0,\n                                                temperature=temperature, score_corrector=corrector, corrector_kwargs=corrector_kwargs,\n                                                x_t=x_T)\n        t1 = time.time()\n\n        if ddim_use_x0_pred:\n            sample = intermediates['pred_x0'][-1]\n\n    x_sample = model.decode_first_stage(sample)\n\n    try:\n        x_sample_noquant = model.decode_first_stage(sample, force_not_quantize=True)\n        log[\"sample_noquant\"] = x_sample_noquant\n        log[\"sample_diff\"] = torch.abs(x_sample_noquant - x_sample)\n    except:\n        pass\n\n    log[\"sample\"] = x_sample\n    log[\"time\"] = t1 - t0\n\n    return log\n", "path": "modules/ldsr_model_arch.py"}]}