diff --git a/frontend_tests/node_tests/message_store.js b/frontend_tests/node_tests/message_store.js
index d5f2140adf..9b88267321 100644
--- a/frontend_tests/node_tests/message_store.js
+++ b/frontend_tests/node_tests/message_store.js
@@ -52,6 +52,9 @@ var message_store = require('js/message_store.js');
     message_store.insert_recent_private_message('2', 2001);
     message_store.insert_recent_private_message('1', 3001);
 
+    // try to backdate user1's timestamp
+    message_store.insert_recent_private_message('1', 555);
+
     assert.deepEqual(message_store.recent_private_messages, [
         {user_ids_string: '1', timestamp: 3001},
         {user_ids_string: '2', timestamp: 2001},
diff --git a/static/js/message_store.js b/static/js/message_store.js
index 2b34661e65..74ce6104a0 100644
--- a/static/js/message_store.js
+++ b/static/js/message_store.js
@@ -71,22 +71,37 @@ exports.process_message_for_recent_private_messages = function (message) {
     exports.insert_recent_private_message(user_ids_string, message.timestamp);
 };
 
-exports.insert_recent_private_message = function (user_ids_string, timestamp) {
-    // If this conversation is already tracked, we'll replace with new timestamp,
-    // so remove it from the current list.
-    exports.recent_private_messages = _.filter(exports.recent_private_messages,
-                                               function (recent_pm) {
-        return recent_pm.user_ids_string !== user_ids_string;
-    });
+exports.insert_recent_private_message = (function () {
+    var recent_timestamps = new Dict({fold_case: true}); // key is user_ids_string
 
-    var new_conversation = {user_ids_string: user_ids_string,
-                            timestamp: timestamp};
+    return function (user_ids_string, timestamp) {
+        var conversation = recent_timestamps.get(user_ids_string);
 
-    exports.recent_private_messages.push(new_conversation);
-    exports.recent_private_messages.sort(function (a, b) {
-        return b.timestamp - a.timestamp;
-    });
-};
+        if (conversation === undefined) {
+            // This is a new user, so create a new object.
+            conversation = {
+                user_ids_string: user_ids_string,
+                timestamp: timestamp,
+            };
+            recent_timestamps.set(user_ids_string, conversation);
+
+            // Optimistically insert the new message at the front, since that
+            // is usually where it belongs, but we'll re-sort.
+            exports.recent_private_messages.unshift(conversation);
+        } else {
+            if (conversation.timestamp >= timestamp) {
+                return; // don't backdate our conversation
+            }
+
+            // update our timestamp
+            conversation.timestamp = timestamp;
+        }
+
+        exports.recent_private_messages.sort(function (a, b) {
+            return b.timestamp - a.timestamp;
+        });
+    };
+}());
 
 exports.set_topic_edit_properties = function (message) {
     message.always_visible_topic_edit = false;