diff --git a/mlir/docs/Dialects/ArmSME.md b/mlir/docs/Dialects/ArmSME.md
new file mode 100644
index 00000000000000..ab7c9ffe7aa92f
--- /dev/null
+++ b/mlir/docs/Dialects/ArmSME.md
@@ -0,0 +1,16 @@
+# 'ArmSME' Dialect
+
+Basic dialect to target Arm SME architectures This dialect contains the
+definitions necessary to target Arm SME scalable matrix operations.
+
+## References
+* https://developer.arm.com/documentation/ddi0616
+* https://developer.arm.com/documentation/ddi0602/2023-03/SME-Instructions
+
+## Operations
+
+[include "Dialects/ArmSMEOps.md"]
+
+## Operations for LLVM IR Intrinsics
+
+[include "Dialects/ArmSMEIntrinsicOps.md"]
diff --git a/mlir/include/mlir/Dialect/ArmSME/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/ArmSME/IR/CMakeLists.txt
index e0d66ab853e55f..9319a8042c93f4 100644
--- a/mlir/include/mlir/Dialect/ArmSME/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/ArmSME/IR/CMakeLists.txt
@@ -1,5 +1,4 @@
 add_mlir_dialect(ArmSME arm_sme ArmSME)
-add_mlir_doc(ArmSME ArmSME Dialects/ -gen-dialect-doc -dialect=arm_sme)
 
 # Generate declarations and definitions of ArmSME Ops
 set(LLVM_TARGET_DEFINITIONS ArmSMEOps.td)
@@ -22,3 +21,7 @@ mlir_tablegen(ArmSMEIntrinsicOps.h.inc -gen-op-decls)
 mlir_tablegen(ArmSMEIntrinsicOps.cpp.inc -gen-op-defs)
 mlir_tablegen(ArmSMEIntrinsicConversions.inc -gen-llvmir-conversions)
 add_public_tablegen_target(MLIRArmSMEIntrinsicOpsIncGen)
+
+# Generate the docs
+add_mlir_doc(ArmSMEOps ArmSMEOps Dialects/ -gen-op-doc)
+add_mlir_doc(ArmSMEIntrinsicOps ArmSMEIntrinsicOps Dialects/ -gen-op-doc)