diff --git a/doc/whats_new/v0.20.rst b/doc/whats_new/v0.20.rst
index a916a9bbd644a..bffe83835187e 100644
--- a/doc/whats_new/v0.20.rst
+++ b/doc/whats_new/v0.20.rst
@@ -22,6 +22,7 @@ random sampling procedures.
 - :class:`neural_network.BaseMultilayerPerceptron` (bug fix)
 - :class:`neural_network.MLPRegressor` (bug fix)
 - :class:`neural_network.MLPClassifier` (bug fix)
+- :class:`linear_model.ARDRegression` (bug fix)
 
 Details are listed in the changelog below.
 
@@ -140,6 +141,10 @@ Classifiers and regressors
   broken when setting ``normalize=False``.
   by `Alexandre Gramfort`_.
 
+- Fixed a bug in :class:`linear_model.ARDRegression` which caused incorrectly
+  updated estimates for the standard deviation and the coefficients.
+  :issue:`10153` by :user:`Jörg Döpfert <jdoepfert>`.
+
 Decomposition, manifold learning and clustering
 
 - Fix for uninformative error in :class:`decomposition.IncrementalPCA`:
diff --git a/sklearn/linear_model/bayes.py b/sklearn/linear_model/bayes.py
index e754613cda381..9fda4f88cfc27 100644
--- a/sklearn/linear_model/bayes.py
+++ b/sklearn/linear_model/bayes.py
@@ -469,9 +469,8 @@ def fit(self, X, y):
         self.scores_ = list()
         coef_old_ = None
 
-        # Iterative procedure of ARDRegression
-        for iter_ in range(self.n_iter):
-            # Compute mu and sigma (using Woodbury matrix identity)
+        # Compute sigma and mu (using Woodbury matrix identity)
+        def update_sigma(X, alpha_, lambda_, keep_lambda, n_samples):
             sigma_ = pinvh(np.eye(n_samples) / alpha_ +
                            np.dot(X[:, keep_lambda] *
                            np.reshape(1. / lambda_[keep_lambda], [1, -1]),
@@ -481,8 +480,17 @@ def fit(self, X, y):
             sigma_ = - np.dot(np.reshape(1. / lambda_[keep_lambda], [-1, 1]) *
                               X[:, keep_lambda].T, sigma_)
             sigma_.flat[::(sigma_.shape[1] + 1)] += 1. / lambda_[keep_lambda]
+            return sigma_
+
+        def update_coeff(X, y, coef_, alpha_, keep_lambda, sigma_):
             coef_[keep_lambda] = alpha_ * np.dot(
                 sigma_, np.dot(X[:, keep_lambda].T, y))
+            return coef_
+
+        # Iterative procedure of ARDRegression
+        for iter_ in range(self.n_iter):
+            sigma_ = update_sigma(X, alpha_, lambda_, keep_lambda, n_samples)
+            coef_ = update_coeff(X, y, coef_, alpha_, keep_lambda, sigma_)
 
             # Update alpha and lambda
             rmse_ = np.sum((y - np.dot(X, coef_)) ** 2)
@@ -513,6 +521,10 @@ def fit(self, X, y):
                 break
             coef_old_ = np.copy(coef_)
 
+        # update sigma and mu using updated parameters from the last iteration
+        sigma_ = update_sigma(X, alpha_, lambda_, keep_lambda, n_samples)
+        coef_ = update_coeff(X, y, coef_, alpha_, keep_lambda, sigma_)
+
         self.coef_ = coef_
         self.alpha_ = alpha_
         self.sigma_ = sigma_
diff --git a/sklearn/linear_model/tests/test_bayes.py b/sklearn/linear_model/tests/test_bayes.py
index 5337c0a19c5cf..443f856fa7285 100644
--- a/sklearn/linear_model/tests/test_bayes.py
+++ b/sklearn/linear_model/tests/test_bayes.py
@@ -9,6 +9,7 @@
 from sklearn.utils.testing import assert_array_almost_equal
 from sklearn.utils.testing import assert_almost_equal
 from sklearn.utils.testing import assert_array_less
+from sklearn.utils.testing import assert_equal
 from sklearn.utils.testing import SkipTest
 from sklearn.utils import check_random_state
 from sklearn.linear_model.bayes import BayesianRidge, ARDRegression
@@ -110,6 +111,21 @@ def test_std_bayesian_ridge_ard_with_constant_input():
         assert_array_less(y_std, expected_upper_boundary)
 
 
+def test_update_of_sigma_in_ard():
+    # Checks that `sigma_` is updated correctly after the last iteration
+    # of the ARDRegression algorithm. See issue #10128.
+    X = np.array([[1, 0],
+                  [0, 0]])
+    y = np.array([0, 0])
+    clf = ARDRegression(n_iter=1)
+    clf.fit(X, y)
+    # With the inputs above, ARDRegression prunes one of the two coefficients
+    # in the first iteration. Hence, the expected shape of `sigma_` is (1, 1).
+    assert_equal(clf.sigma_.shape, (1, 1))
+    # Ensure that no error is thrown at prediction stage
+    clf.predict(X, return_std=True)
+
+
 def test_toy_ard_object():
     # Test BayesianRegression ARD classifier
     X = np.array([[1], [2], [3]])