diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index 1cd62184ccbe3..4c454e81b232d 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -669,7 +669,14 @@ static void AdjustIntegerStringWithScale(int32_t scale, std::string* str) {
     //               adjusted_exponent = -7
     // After inserting decimal point: *str = "-1.23"
     // After appending exponent: *str = "-1.23E-7"
-    str->insert(str->begin() + 1 + is_negative_offset, '.');
+    // Example 3:
+    // Precondition: *str = "0", is_negative_offset = 0, num_digits = 1, scale = -1,
+    //               adjusted_exponent = 1
+    // After inserting decimal point: *str = "0" // Not inserted
+    // After appending exponent: *str = "0E+1"
+    if (num_digits > 1) {
+      str->insert(str->begin() + 1 + is_negative_offset, '.');
+    }
     str->push_back('E');
     if (adjusted_exponent >= 0) {
       str->push_back('+');
diff --git a/cpp/src/arrow/util/decimal_test.cc b/cpp/src/arrow/util/decimal_test.cc
index e2e9e2901957d..d2f8ae3b7aad2 100644
--- a/cpp/src/arrow/util/decimal_test.cc
+++ b/cpp/src/arrow/util/decimal_test.cc
@@ -711,22 +711,22 @@ struct ToStringTestParam {
 };
 
 static const ToStringTestParam kToStringTestData[] = {
-    {0, -1, "0.E+1"},
+    {0, -1, "0E+1"},
     {0, 0, "0"},
     {0, 1, "0.0"},
     {0, 6, "0.000000"},
-    {2, 7, "2.E-7"},
-    {2, -1, "2.E+1"},
+    {2, 7, "2E-7"},
+    {2, -1, "2E+1"},
     {2, 0, "2"},
     {2, 1, "0.2"},
     {2, 6, "0.000002"},
-    {-2, 7, "-2.E-7"},
-    {-2, 7, "-2.E-7"},
-    {-2, -1, "-2.E+1"},
+    {-2, 7, "-2E-7"},
+    {-2, 7, "-2E-7"},
+    {-2, -1, "-2E+1"},
     {-2, 0, "-2"},
     {-2, 1, "-0.2"},
     {-2, 6, "-0.000002"},
-    {-2, 7, "-2.E-7"},
+    {-2, 7, "-2E-7"},
     {123, -3, "1.23E+5"},
     {123, -1, "1.23E+3"},
     {123, 1, "12.3"},
diff --git a/cpp/src/arrow/util/formatting_util_test.cc b/cpp/src/arrow/util/formatting_util_test.cc
index f1846e279aca2..457bb4c88d344 100644
--- a/cpp/src/arrow/util/formatting_util_test.cc
+++ b/cpp/src/arrow/util/formatting_util_test.cc
@@ -318,22 +318,22 @@ void TestDecimalFormatter() {
 
   // Borrow from Decimal::ToString test
   const auto decimalTestData = std::vector<TestParam>{
-      {0, -1, "0.E+1"},
+      {0, -1, "0E+1"},
       {0, 0, "0"},
       {0, 1, "0.0"},
       {0, 6, "0.000000"},
-      {2, 7, "2.E-7"},
-      {2, -1, "2.E+1"},
+      {2, 7, "2E-7"},
+      {2, -1, "2E+1"},
       {2, 0, "2"},
       {2, 1, "0.2"},
       {2, 6, "0.000002"},
-      {-2, 7, "-2.E-7"},
-      {-2, 7, "-2.E-7"},
-      {-2, -1, "-2.E+1"},
+      {-2, 7, "-2E-7"},
+      {-2, 7, "-2E-7"},
+      {-2, -1, "-2E+1"},
       {-2, 0, "-2"},
       {-2, 1, "-0.2"},
       {-2, 6, "-0.000002"},
-      {-2, 7, "-2.E-7"},
+      {-2, 7, "-2E-7"},
       {123, -3, "1.23E+5"},
       {123, -1, "1.23E+3"},
       {123, 1, "12.3"},
diff --git a/ruby/red-arrow/lib/arrow/decimal128-array.rb b/ruby/red-arrow/lib/arrow/decimal128-array.rb
index 528c878a859b5..a5ee53be7b229 100644
--- a/ruby/red-arrow/lib/arrow/decimal128-array.rb
+++ b/ruby/red-arrow/lib/arrow/decimal128-array.rb
@@ -18,9 +18,7 @@
 module Arrow
   class Decimal128Array
     def get_value(i)
-      string = format_value(i)
-      string.sub!(".E", ".0E") if string.include?(".E")
-      BigDecimal(string)
+      BigDecimal(format_value(i))
     end
   end
 end
diff --git a/ruby/red-arrow/lib/arrow/decimal256-array.rb b/ruby/red-arrow/lib/arrow/decimal256-array.rb
index 32841ca4862f5..8c2306dfe3627 100644
--- a/ruby/red-arrow/lib/arrow/decimal256-array.rb
+++ b/ruby/red-arrow/lib/arrow/decimal256-array.rb
@@ -19,9 +19,7 @@ module Arrow
   class Decimal256Array
     # @since 3.0.0
     def get_value(i)
-      string = format_value(i)
-      string.sub!(".E", ".0E") if string.include?(".E")
-      BigDecimal(string)
+      BigDecimal(format_value(i))
     end
   end
 end