diff --git a/plugins/removeHiddenElems.js b/plugins/removeHiddenElems.js
index ad58324d8..38b4f7c40 100644
--- a/plugins/removeHiddenElems.js
+++ b/plugins/removeHiddenElems.js
@@ -15,6 +15,7 @@ const {
 } = require('../lib/xast.js');
 const { collectStylesheet, computeStyle } = require('../lib/style.js');
 const { parsePathData } = require('../lib/path.js');
+const { hasScripts } = require('../lib/svgo/tools.js');
 
 const nonRendering = elemsGroups.nonRendering;
 
@@ -74,6 +75,21 @@ exports.fn = (root, params) => {
    */
   const removedDefIds = new Set();
 
+  /**
+   * @type {Map<XastElement, XastParent>}
+   */
+  const allDefs = new Map();
+
+  /**
+   * @type {Map<string, Array<{ node: XastElement, parentNode: XastParent }>>}
+   */
+  const referencesById = new Map();
+
+  /**
+   * If styles are present, we can't be sure if a definition is unused or not
+   */
+  let deoptimized = false;
+
   /**
    * @param {XastChild} node
    * @param {XastParent} parentNode
@@ -123,6 +139,33 @@ exports.fn = (root, params) => {
   return {
     element: {
       enter: (node, parentNode) => {
+        if (
+          (node.name === 'style' && node.children.length !== 0) ||
+          hasScripts(node)
+        ) {
+          deoptimized = true;
+          return;
+        }
+
+        if (node.name === 'defs') {
+          allDefs.set(node, parentNode);
+        }
+
+        if (node.name === 'use') {
+          for (const attr of Object.keys(node.attributes)) {
+            if (attr !== 'href' && !attr.endsWith(':href')) continue;
+            const value = node.attributes[attr];
+            const id = value.slice(1);
+
+            let refs = referencesById.get(id);
+            if (!refs) {
+              refs = [];
+              referencesById.set(id, refs);
+            }
+            refs.push({ node, parentNode });
+          }
+        }
+
         // Removes hidden elements
         // https://www.w3schools.com/cssref/pr_class_visibility.asp
         const computedStyle = computeStyle(stylesheet, node);
@@ -350,46 +393,41 @@ exports.fn = (root, params) => {
           removeElement(node, parentNode);
         }
       },
-
-      exit: (node, parentNode) => {
-        if (node.name === 'defs' && node.children.length === 0) {
-          removeElement(node, parentNode);
-          return;
-        }
-
-        if (node.name === 'use') {
-          const referencesRemovedDef = Object.entries(node.attributes).some(
-            ([attrKey, attrValue]) =>
-              (attrKey === 'href' || attrKey.endsWith(':href')) &&
-              removedDefIds.has(
-                attrValue.slice(attrValue.indexOf('#') + 1).trim()
-              )
-          );
-
-          if (referencesRemovedDef) {
-            detachNodeFromParent(node, parentNode);
+    },
+    root: {
+      exit: () => {
+        for (const id of removedDefIds) {
+          const refs = referencesById.get(id);
+          if (refs) {
+            for (const { node, parentNode } of refs) {
+              detachNodeFromParent(node, parentNode);
+            }
           }
-
-          return;
         }
 
-        if (node.name === 'svg' && parentNode.type === 'root') {
+        if (!deoptimized) {
           for (const [
             nonRenderedNode,
             nonRenderedParent,
           ] of nonRenderedNodes.entries()) {
+            const id = nonRenderedNode.attributes.id;
             const selector = referencesProps
-              .map(
-                (attr) => `[${attr}="url(#${nonRenderedNode.attributes.id})"]`
-              )
+              .map((attr) => `[${attr}="url(#${id})"]`)
+              .concat(`[href="#${id}"]`, `[xlink\\:href="#${id}"]`)
               .join(',');
 
             const element = querySelector(root, selector);
             if (element == null) {
-              detachNodeFromParent(node, nonRenderedParent);
+              detachNodeFromParent(nonRenderedNode, nonRenderedParent);
             }
           }
         }
+
+        for (const [node, parentNode] of allDefs.entries()) {
+          if (node.children.length === 0) {
+            detachNodeFromParent(node, parentNode);
+          }
+        }
       },
     },
   };
diff --git a/test/plugins/removeHiddenElems.14.svg b/test/plugins/removeHiddenElems.14.svg
new file mode 100644
index 000000000..a87a84bc2
--- /dev/null
+++ b/test/plugins/removeHiddenElems.14.svg
@@ -0,0 +1,14 @@
+Remove unused defs
+
+===
+
+<svg xmlns="http://www.w3.org/2000/svg">
+    <defs>
+        <linearGradient id="a">
+        </linearGradient>
+    </defs>
+</svg>
+
+@@@
+
+<svg xmlns="http://www.w3.org/2000/svg"/>
diff --git a/test/plugins/removeHiddenElems.15.svg b/test/plugins/removeHiddenElems.15.svg
new file mode 100644
index 000000000..e469b102d
--- /dev/null
+++ b/test/plugins/removeHiddenElems.15.svg
@@ -0,0 +1,20 @@
+Don't remove used defs
+
+===
+
+<svg xmlns="http://www.w3.org/2000/svg">
+    <rect fill="url(#a)" width="64" height="64"/>
+    <defs>
+        <linearGradient id="a">
+        </linearGradient>
+    </defs>
+</svg>
+
+@@@
+
+<svg xmlns="http://www.w3.org/2000/svg">
+    <rect fill="url(#a)" width="64" height="64"/>
+    <defs>
+        <linearGradient id="a"/>
+    </defs>
+</svg>
\ No newline at end of file