From f174fb29228a04955fb951b32b0bab79e33ec2b8 Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Fri, 7 Oct 2022 05:21:49 +0300
Subject: [PATCH 01/21] add xformers attention

---
 modules/sd_hijack_optimizations.py | 39 +++++++++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)

diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py
index ea4cfdfcd66..da1b76e1c58 100644
--- a/modules/sd_hijack_optimizations.py
+++ b/modules/sd_hijack_optimizations.py
@@ -1,7 +1,9 @@
 import math
 import torch
 from torch import einsum
-
+import xformers.ops
+import functorch
+xformers._is_functorch_available=True
 from ldm.util import default
 from einops import rearrange
 
@@ -92,6 +94,41 @@ def split_cross_attention_forward(self, x, context=None, mask=None):
 
     return self.to_out(r2)
 
+def _maybe_init(self, x):
+    """
+    Initialize the attention operator, if required We expect the head dimension to be exposed here, meaning that x
+    : B, Head, Length
+    """
+    if self.attention_op is not None:
+        return
+    _, M, K = x.shape
+    try:
+        self.attention_op = xformers.ops.AttentionOpDispatch(
+            dtype=x.dtype,
+            device=x.device,
+            k=K,
+            attn_bias_type=type(None),
+            has_dropout=False,
+            kv_len=M,
+            q_len=M,
+        ).op
+    except NotImplementedError as err:
+        raise NotImplementedError(f"Please install xformers with the flash attention / cutlass components.\n{err}")
+
+def xformers_attention_forward(self, x, context=None, mask=None):
+    h = self.heads
+    q_in = self.to_q(x)
+    context = default(context, x)
+    k_in = self.to_k(context)
+    v_in = self.to_v(context)
+    q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h=h), (q_in, k_in, v_in))
+    del q_in, k_in, v_in
+    self._maybe_init(q)
+    out = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=None, op=self.attention_op)
+
+    out = rearrange(out, '(b h) n d -> b n (h d)', h=h)
+    return self.to_out(out)
+
 def cross_attention_attnblock_forward(self, x):
         h_ = x
         h_ = self.norm(h_)

From 2eb911b056ce6ff4434f673366782ed34f2b2f12 Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Fri, 7 Oct 2022 05:22:28 +0300
Subject: [PATCH 02/21] Update sd_hijack.py

---
 modules/sd_hijack.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py
index a6fa890c424..6221ed5ac57 100644
--- a/modules/sd_hijack.py
+++ b/modules/sd_hijack.py
@@ -20,12 +20,17 @@
 
 
 def apply_optimizations():
-    ldm.modules.diffusionmodules.model.nonlinearity = silu
-
     if cmd_opts.opt_split_attention_v1:
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.split_cross_attention_forward_v1
-    elif not cmd_opts.disable_opt_split_attention and (cmd_opts.opt_split_attention or torch.cuda.is_available()):
-        ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.split_cross_attention_forward
+    if cmd_opts.opt_split_attention:
+        ldm.modules.attention_CrossAttention_forward = sd_hijack_optimizations.split_cross_attention_forward
+        ldm.modules.diffusionmodules.model.nonlinearity = sd_hijack_optimizations.nonlinearity_hijack
+        ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.cross_attention_attnblock_forward
+    elif not cmd_opts.disable_opt_xformers_attention:
+        ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.xformers_attention_forward
+        ldm.modules.attention.CrossAttention._maybe_init = sd_hijack_optimizations._maybe_init
+        ldm.modules.attention.CrossAttention.attention_op = None
+        ldm.modules.diffusionmodules.model.nonlinearity = sd_hijack_optimizations.nonlinearity_hijack
         ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.cross_attention_attnblock_forward
 
 

From da4ab2707b4cb0611cf181ba248a271d1937433e Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Fri, 7 Oct 2022 05:23:06 +0300
Subject: [PATCH 03/21] Update shared.py

---
 modules/shared.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/modules/shared.py b/modules/shared.py
index 25bb6e6c944..8cc3b2fe28d 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -43,6 +43,7 @@
 parser.add_argument("--scunet-models-path", type=str, help="Path to directory with ScuNET model file(s).", default=os.path.join(models_path, 'ScuNET'))
 parser.add_argument("--swinir-models-path", type=str, help="Path to directory with SwinIR model file(s).", default=os.path.join(models_path, 'SwinIR'))
 parser.add_argument("--ldsr-models-path", type=str, help="Path to directory with LDSR model file(s).", default=os.path.join(models_path, 'LDSR'))
+parser.add_argument("--disable-opt-xformers-attention", action='store_true', help="force-disables xformers attention optimization")
 parser.add_argument("--opt-split-attention", action='store_true', help="force-enables cross-attention layer optimization. By default, it's on for torch.cuda and off for other torch devices.")
 parser.add_argument("--disable-opt-split-attention", action='store_true', help="force-disables cross-attention layer optimization")
 parser.add_argument("--opt-split-attention-v1", action='store_true', help="enable older version of split attention optimization that does not consume all the VRAM it can find")

From cd8bb597c6bcb6c59b538b7a1ab8f2face764fc5 Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Fri, 7 Oct 2022 05:23:25 +0300
Subject: [PATCH 04/21] Update requirements.txt

---
 requirements.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/requirements.txt b/requirements.txt
index 631fe616adb..304a066a30a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -23,3 +23,5 @@ resize-right
 torchdiffeq
 kornia
 lark
+functorch
+#xformers?

From 35d6b231628d18d53d166c3a92fea1523e88d51e Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Fri, 7 Oct 2022 05:31:53 +0300
Subject: [PATCH 05/21] Update sd_hijack.py

---
 modules/sd_hijack.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py
index 6221ed5ac57..a006c0a3b86 100644
--- a/modules/sd_hijack.py
+++ b/modules/sd_hijack.py
@@ -20,17 +20,16 @@
 
 
 def apply_optimizations():
+    ldm.modules.diffusionmodules.model.nonlinearity = silu
     if cmd_opts.opt_split_attention_v1:
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.split_cross_attention_forward_v1
     if cmd_opts.opt_split_attention:
         ldm.modules.attention_CrossAttention_forward = sd_hijack_optimizations.split_cross_attention_forward
-        ldm.modules.diffusionmodules.model.nonlinearity = sd_hijack_optimizations.nonlinearity_hijack
         ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.cross_attention_attnblock_forward
     elif not cmd_opts.disable_opt_xformers_attention:
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.xformers_attention_forward
         ldm.modules.attention.CrossAttention._maybe_init = sd_hijack_optimizations._maybe_init
         ldm.modules.attention.CrossAttention.attention_op = None
-        ldm.modules.diffusionmodules.model.nonlinearity = sd_hijack_optimizations.nonlinearity_hijack
         ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.cross_attention_attnblock_forward
 
 

From 5303df24282ba06abb34a423f2967354d37d078e Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Fri, 7 Oct 2022 06:01:14 +0300
Subject: [PATCH 06/21] Update sd_hijack.py

---
 modules/sd_hijack.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py
index a006c0a3b86..ddacb0ad858 100644
--- a/modules/sd_hijack.py
+++ b/modules/sd_hijack.py
@@ -23,10 +23,10 @@ def apply_optimizations():
     ldm.modules.diffusionmodules.model.nonlinearity = silu
     if cmd_opts.opt_split_attention_v1:
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.split_cross_attention_forward_v1
-    if cmd_opts.opt_split_attention:
+    elif cmd_opts.opt_split_attention:
         ldm.modules.attention_CrossAttention_forward = sd_hijack_optimizations.split_cross_attention_forward
         ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.cross_attention_attnblock_forward
-    elif not cmd_opts.disable_opt_xformers_attention:
+    elif not cmd_opts.disable_opt_xformers_attention and not cmd_opts.opt_split_attention:
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.xformers_attention_forward
         ldm.modules.attention.CrossAttention._maybe_init = sd_hijack_optimizations._maybe_init
         ldm.modules.attention.CrossAttention.attention_op = None

From 5e3ff846c56dc8e1d5c76ea04a8f2f74d7da07fc Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Fri, 7 Oct 2022 06:38:01 +0300
Subject: [PATCH 07/21] Update sd_hijack.py

---
 modules/sd_hijack.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py
index ddacb0ad858..cbdb9d3c7fb 100644
--- a/modules/sd_hijack.py
+++ b/modules/sd_hijack.py
@@ -26,7 +26,7 @@ def apply_optimizations():
     elif cmd_opts.opt_split_attention:
         ldm.modules.attention_CrossAttention_forward = sd_hijack_optimizations.split_cross_attention_forward
         ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.cross_attention_attnblock_forward
-    elif not cmd_opts.disable_opt_xformers_attention and not cmd_opts.opt_split_attention:
+    elif not cmd_opts.disable_opt_xformers_attention and not (cmd_opts.opt_split_attention or torch.version.hip):
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.xformers_attention_forward
         ldm.modules.attention.CrossAttention._maybe_init = sd_hijack_optimizations._maybe_init
         ldm.modules.attention.CrossAttention.attention_op = None

From c9cc65b201679ea43c763b0d85e749d40bbc5433 Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Sat, 8 Oct 2022 04:09:18 +0300
Subject: [PATCH 08/21] switch to the proper way of calling xformers

---
 modules/sd_hijack_optimizations.py | 28 +++-------------------------
 1 file changed, 3 insertions(+), 25 deletions(-)

diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py
index da1b76e1c58..7fb4a45e3a7 100644
--- a/modules/sd_hijack_optimizations.py
+++ b/modules/sd_hijack_optimizations.py
@@ -94,39 +94,17 @@ def split_cross_attention_forward(self, x, context=None, mask=None):
 
     return self.to_out(r2)
 
-def _maybe_init(self, x):
-    """
-    Initialize the attention operator, if required We expect the head dimension to be exposed here, meaning that x
-    : B, Head, Length
-    """
-    if self.attention_op is not None:
-        return
-    _, M, K = x.shape
-    try:
-        self.attention_op = xformers.ops.AttentionOpDispatch(
-            dtype=x.dtype,
-            device=x.device,
-            k=K,
-            attn_bias_type=type(None),
-            has_dropout=False,
-            kv_len=M,
-            q_len=M,
-        ).op
-    except NotImplementedError as err:
-        raise NotImplementedError(f"Please install xformers with the flash attention / cutlass components.\n{err}")
-
 def xformers_attention_forward(self, x, context=None, mask=None):
     h = self.heads
     q_in = self.to_q(x)
     context = default(context, x)
     k_in = self.to_k(context)
     v_in = self.to_v(context)
-    q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h=h), (q_in, k_in, v_in))
+    q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b n h d', h=h), (q_in, k_in, v_in))
     del q_in, k_in, v_in
-    self._maybe_init(q)
-    out = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=None, op=self.attention_op)
+    out = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=None)
 
-    out = rearrange(out, '(b h) n d -> b n (h d)', h=h)
+    out = rearrange(out, 'b n h d -> b n (h d)', h=h)
     return self.to_out(out)
 
 def cross_attention_attnblock_forward(self, x):

From b70eaeb2005a5a9593119e7fd32b8072c2a208d5 Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Sat, 8 Oct 2022 04:10:35 +0300
Subject: [PATCH 09/21] delete broken and unnecessary aliases

---
 modules/sd_hijack.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py
index cbdb9d3c7fb..0e99c319287 100644
--- a/modules/sd_hijack.py
+++ b/modules/sd_hijack.py
@@ -21,16 +21,14 @@
 
 def apply_optimizations():
     ldm.modules.diffusionmodules.model.nonlinearity = silu
-    if cmd_opts.opt_split_attention_v1:
+    if not cmd_opts.disable_opt_xformers_attention and not (cmd_opts.opt_split_attention or torch.version.hip):
+        ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.xformers_attention_forward
+        ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.cross_attention_attnblock_forward
+    elif cmd_opts.opt_split_attention_v1:
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.split_cross_attention_forward_v1
     elif cmd_opts.opt_split_attention:
         ldm.modules.attention_CrossAttention_forward = sd_hijack_optimizations.split_cross_attention_forward
         ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.cross_attention_attnblock_forward
-    elif not cmd_opts.disable_opt_xformers_attention and not (cmd_opts.opt_split_attention or torch.version.hip):
-        ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.xformers_attention_forward
-        ldm.modules.attention.CrossAttention._maybe_init = sd_hijack_optimizations._maybe_init
-        ldm.modules.attention.CrossAttention.attention_op = None
-        ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.cross_attention_attnblock_forward
 
 
 def undo_optimizations():

From 5d54f35c583bd5a3b0ee271a862827f1ca81ef09 Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Sat, 8 Oct 2022 11:55:02 +0300
Subject: [PATCH 10/21] add xformers attnblock and hypernetwork support

---
 modules/sd_hijack_optimizations.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py
index 7fb4a45e3a7..c78d58382d5 100644
--- a/modules/sd_hijack_optimizations.py
+++ b/modules/sd_hijack_optimizations.py
@@ -98,8 +98,14 @@ def xformers_attention_forward(self, x, context=None, mask=None):
     h = self.heads
     q_in = self.to_q(x)
     context = default(context, x)
-    k_in = self.to_k(context)
-    v_in = self.to_v(context)
+    hypernetwork = shared.selected_hypernetwork()
+    hypernetwork_layers = (hypernetwork.layers if hypernetwork is not None else {}).get(context.shape[2], None)
+    if hypernetwork_layers is not None:
+        k_in = self.to_k(hypernetwork_layers[0](context))
+        v_in = self.to_v(hypernetwork_layers[1](context))
+    else:
+        k_in = self.to_k(context)
+        v_in = self.to_v(context)
     q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b n h d', h=h), (q_in, k_in, v_in))
     del q_in, k_in, v_in
     out = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=None)
@@ -169,3 +175,13 @@ def cross_attention_attnblock_forward(self, x):
         h3 += x
 
         return h3
+    
+    def xformers_attnblock_forward(self, x):
+        h_ = x
+        h_ = self.norm(h_)
+        q1 = self.q(h_).contiguous()
+        k1 = self.k(h_).contiguous()
+        v = self.v(h_).contiguous()
+        out = xformers.ops.memory_efficient_attention(q1, k1, v)
+        out = self.proj_out(out)
+        return x+out

From 76a616fa6b814c681eaf6edc87eb3001b8c2b6be Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Sat, 8 Oct 2022 11:55:38 +0300
Subject: [PATCH 11/21] Update sd_hijack_optimizations.py

---
 modules/sd_hijack_optimizations.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py
index c78d58382d5..ee58c7e4e64 100644
--- a/modules/sd_hijack_optimizations.py
+++ b/modules/sd_hijack_optimizations.py
@@ -176,7 +176,7 @@ def cross_attention_attnblock_forward(self, x):
 
         return h3
     
-    def xformers_attnblock_forward(self, x):
+def xformers_attnblock_forward(self, x):
         h_ = x
         h_ = self.norm(h_)
         q1 = self.q(h_).contiguous()

From 91d66f5520df416db718103d460550ad495e952d Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Sat, 8 Oct 2022 11:56:01 +0300
Subject: [PATCH 12/21] use new attnblock for xformers path

---
 modules/sd_hijack.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py
index 0e99c319287..3da8c8ce2ad 100644
--- a/modules/sd_hijack.py
+++ b/modules/sd_hijack.py
@@ -23,7 +23,7 @@ def apply_optimizations():
     ldm.modules.diffusionmodules.model.nonlinearity = silu
     if not cmd_opts.disable_opt_xformers_attention and not (cmd_opts.opt_split_attention or torch.version.hip):
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.xformers_attention_forward
-        ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.cross_attention_attnblock_forward
+        ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.xformers_attnblock_forward
     elif cmd_opts.opt_split_attention_v1:
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.split_cross_attention_forward_v1
     elif cmd_opts.opt_split_attention:

From 4201fd14f5769a4cf6723d2bc5495c3c84a2cd00 Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Sat, 8 Oct 2022 14:42:34 +0300
Subject: [PATCH 13/21] install xformers

---
 launch.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/launch.py b/launch.py
index 75edb66a970..f3fbe16a5e1 100644
--- a/launch.py
+++ b/launch.py
@@ -124,6 +124,9 @@ def git_clone(url, dir, name, commithash=None):
 if not is_installed("clip"):
     run_pip(f"install {clip_package}", "clip")
 
+if not is_installed("xformers"):
+    run_pip("install https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/a/xformers-0.0.14.dev0-cp310-cp310-win_amd64.whl", "xformers")
+
 os.makedirs(dir_repos, exist_ok=True)
 
 git_clone("https://github.com/CompVis/stable-diffusion.git", repo_dir('stable-diffusion'), "Stable Diffusion", stable_diffusion_commit_hash)

From 3f166be1b60ff2ab33a6d2646809ec3f48796303 Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Sat, 8 Oct 2022 14:42:50 +0300
Subject: [PATCH 14/21] Update requirements.txt

---
 requirements.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 304a066a30a..81641d68fa1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -24,4 +24,3 @@ torchdiffeq
 kornia
 lark
 functorch
-#xformers?

From d0e85873ac72416d32dee8720dc9e93ab3d3e236 Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Sat, 8 Oct 2022 16:13:26 +0300
Subject: [PATCH 15/21] check for OS and env variable

---
 launch.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/launch.py b/launch.py
index f3fbe16a5e1..a2089b3bcef 100644
--- a/launch.py
+++ b/launch.py
@@ -4,6 +4,7 @@
 import sys
 import importlib.util
 import shlex
+import platform
 
 dir_repos = "repositories"
 dir_tmp = "tmp"
@@ -31,6 +32,7 @@ def extract_arg(args, name):
 
 
 args, skip_torch_cuda_test = extract_arg(args, '--skip-torch-cuda-test')
+args, xformers = extract_arg(args, '--xformers')
 
 
 def repo_dir(name):
@@ -124,8 +126,11 @@ def git_clone(url, dir, name, commithash=None):
 if not is_installed("clip"):
     run_pip(f"install {clip_package}", "clip")
 
-if not is_installed("xformers"):
-    run_pip("install https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/a/xformers-0.0.14.dev0-cp310-cp310-win_amd64.whl", "xformers")
+if not is_installed("xformers") and xformers:
+    if platform.system() == "Windows":
+        run_pip("install https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/a/xformers-0.0.14.dev0-cp310-cp310-win_amd64.whl", "xformers")
+    elif:
+        run_pip("install xformers", "xformers")
 
 os.makedirs(dir_repos, exist_ok=True)
 

From 26b459a3799c5cdf71ca8ed5315a99f69c69f02c Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Sat, 8 Oct 2022 16:20:04 +0300
Subject: [PATCH 16/21] default to split attention if cuda is available and
 xformers is not

---
 modules/sd_hijack.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py
index 3da8c8ce2ad..04adcf03546 100644
--- a/modules/sd_hijack.py
+++ b/modules/sd_hijack.py
@@ -21,12 +21,12 @@
 
 def apply_optimizations():
     ldm.modules.diffusionmodules.model.nonlinearity = silu
-    if not cmd_opts.disable_opt_xformers_attention and not (cmd_opts.opt_split_attention or torch.version.hip):
+    if not cmd_opts.disable_opt_xformers_attention and not (cmd_opts.opt_split_attention or torch.version.hip or shared.xformers_available):
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.xformers_attention_forward
         ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.xformers_attnblock_forward
     elif cmd_opts.opt_split_attention_v1:
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.split_cross_attention_forward_v1
-    elif cmd_opts.opt_split_attention:
+    elif cmd_opts.opt_split_attention or torch.cuda.is_available():
         ldm.modules.attention_CrossAttention_forward = sd_hijack_optimizations.split_cross_attention_forward
         ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.cross_attention_attnblock_forward
 

From ddfa9a97865c732193023a71521c5b7b53d8571b Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Sat, 8 Oct 2022 16:20:41 +0300
Subject: [PATCH 17/21] add xformers_available shared variable

---
 modules/shared.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/shared.py b/modules/shared.py
index 8cc3b2fe28d..6ed4b802120 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -74,7 +74,7 @@
 
 batch_cond_uncond = cmd_opts.always_batch_cond_uncond or not (cmd_opts.lowvram or cmd_opts.medvram)
 parallel_processing_allowed = not cmd_opts.lowvram and not cmd_opts.medvram
-
+xformers_available = False
 config_filename = cmd_opts.ui_settings_file
 
 

From 69d0053583757ce2942d62de81e8b89e6be07840 Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Sat, 8 Oct 2022 16:21:40 +0300
Subject: [PATCH 18/21] update sd_hijack_opt to respect new env variables

---
 modules/sd_hijack_optimizations.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py
index ee58c7e4e64..be09ec8f4f5 100644
--- a/modules/sd_hijack_optimizations.py
+++ b/modules/sd_hijack_optimizations.py
@@ -1,9 +1,14 @@
 import math
 import torch
 from torch import einsum
-import xformers.ops
-import functorch
-xformers._is_functorch_available=True
+try:
+    import xformers.ops
+    import functorch
+    xformers._is_functorch_available = True
+    shared.xformers_available = True
+except:
+    print('Cannot find xformers, defaulting to split attention. Try setting --xformers in your webui-user file if you wish to install it.')
+    continue
 from ldm.util import default
 from einops import rearrange
 

From ca5f0f149c29c344a6badd055b15b5e5fcd6e938 Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Sat, 8 Oct 2022 16:22:38 +0300
Subject: [PATCH 19/21] Update launch.py

---
 launch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/launch.py b/launch.py
index a2089b3bcef..a592e1ba7a4 100644
--- a/launch.py
+++ b/launch.py
@@ -129,7 +129,7 @@ def git_clone(url, dir, name, commithash=None):
 if not is_installed("xformers") and xformers:
     if platform.system() == "Windows":
         run_pip("install https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/a/xformers-0.0.14.dev0-cp310-cp310-win_amd64.whl", "xformers")
-    elif:
+    elif platform.system() == "Linux":
         run_pip("install xformers", "xformers")
 
 os.makedirs(dir_repos, exist_ok=True)

From 7ffea1507813540b8cd9e73feb7bf23de1ac4e27 Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Sat, 8 Oct 2022 16:24:06 +0300
Subject: [PATCH 20/21] Update requirements_versions.txt

---
 requirements_versions.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements_versions.txt b/requirements_versions.txt
index fdff2687873..fec3e9d5b72 100644
--- a/requirements_versions.txt
+++ b/requirements_versions.txt
@@ -22,3 +22,4 @@ resize-right==0.0.2
 torchdiffeq==0.2.3
 kornia==0.6.7
 lark==1.1.2
+functorch==0.2.1

From 970de9ee6891ff586821d0d80dde01c2f6c681b3 Mon Sep 17 00:00:00 2001
From: C43H66N12O12S2 <36072735+C43H66N12O12S2@users.noreply.github.com>
Date: Sat, 8 Oct 2022 16:29:43 +0300
Subject: [PATCH 21/21] Update sd_hijack.py

---
 modules/sd_hijack.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py
index 04adcf03546..5b30539fefc 100644
--- a/modules/sd_hijack.py
+++ b/modules/sd_hijack.py
@@ -21,7 +21,7 @@
 
 def apply_optimizations():
     ldm.modules.diffusionmodules.model.nonlinearity = silu
-    if not cmd_opts.disable_opt_xformers_attention and not (cmd_opts.opt_split_attention or torch.version.hip or shared.xformers_available):
+    if not cmd_opts.disable_opt_xformers_attention and not (cmd_opts.opt_split_attention or torch.version.hip) and shared.xformers_available:
         ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.xformers_attention_forward
         ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.xformers_attnblock_forward
     elif cmd_opts.opt_split_attention_v1: