Also remove duplicates from the history file.

When $history_remove_dups is set, remove duplicates from the history
file when it is periodically compacted.

doc/manual.xml.head

@@ -587,7 +587,8 @@ bind editor <delete> backspace
 Mutt maintains a history for the built-in editor.  The number of items
 is controlled by the <link linkend="history">$history</link> variable
 and can be made persistent using an external file specified using <link
-linkend="history-file">$history_file</link>.  You may cycle through them
+linkend="history-file">$history_file</link> and <link
+linkend="save-history">$save_history</link>.  You may cycle through them
 at an editor prompt by using the <literal>&lt;history-up&gt;</literal>
 and/or <literal>&lt;history-down&gt;</literal> commands.  Mutt will
 remember the currently entered text as you cycle through history, and

history.c

@@ -23,6 +23,8 @@
 #include "mutt.h"
 #include "history.h"
 
+#include <stdint.h>
+
 /* This history ring grows from 0..HistSize, with last marking the
  * where new entries go:
  *         0        the oldest entry in the ring
@@ -132,22 +134,69 @@ void mutt_read_histfile (void)
   FREE (&linebuf);
 }
 
+static int dup_hash_dec (HASH *dup_hash, char *s)
+{
+  struct hash_elem *elem;
+  uintptr_t count;
+
+  elem = hash_find_elem (dup_hash, s);
+  if (!elem)
+    return -1;
+
+  count = (uintptr_t)elem->data;
+  if (count <= 1)
+  {
+    hash_delete (dup_hash, s, NULL, NULL);
+    return 0;
+  }
+
+  count--;
+  elem->data = (void *)count;
+  return count;
+}
+
+static int dup_hash_inc (HASH *dup_hash, char *s)
+{
+  struct hash_elem *elem;
+  uintptr_t count;
+
+  elem = hash_find_elem (dup_hash, s);
+  if (!elem)
+  {
+    count = 1;
+    hash_insert (dup_hash, s, (void *)count);
+    return count;
+  }
+
+  count = (uintptr_t)elem->data;
+  count++;
+  elem->data = (void *)count;
+  return count;
+}
+
 static void shrink_histfile (void)
 {
   char tmpfname[_POSIX_PATH_MAX];
   FILE *f, *tmp = NULL;
   int n[HC_LAST] = { 0 };
-  int line, hclass;
-  char *linebuf = NULL;
+  int line, hclass, read;
+  char *linebuf = NULL, *p;
   size_t buflen;
+  int regen_file = 0;
+  HASH *dup_hashes[HC_LAST] = { 0 };
 
   if ((f = fopen (HistFile, "r")) == NULL)
     return;
 
+  if (option (OPTHISTREMOVEDUPS))
+    for (hclass = 0; hclass < HC_LAST; hclass++)
+      dup_hashes[hclass] = hash_create (MAX (10, SaveHist * 2), MUTT_HASH_STRDUP_KEYS);
+
   line = 0;
   while ((linebuf = mutt_read_line (linebuf, &buflen, f, &line, 0)) != NULL)
   {
-    if (sscanf (linebuf, "%d", &hclass) < 1 || hclass < 0)
+    if (sscanf (linebuf, "%d:%n", &hclass, &read) < 1 || read == 0 ||
+        *(p = linebuf + strlen (linebuf) - 1) != '|' || hclass < 0)
     {
       mutt_error (_("Bad history file format (line %d)"), line);
       goto cleanup;
@@ -155,32 +204,49 @@ static void shrink_histfile (void)
     /* silently ignore too high class (probably newer mutt) */
     if (hclass >= HC_LAST)
       continue;
+    *p = '\0';
+    if (option (OPTHISTREMOVEDUPS) &&
+        (dup_hash_inc (dup_hashes[hclass], linebuf + read) > 1))
+    {
+      regen_file = 1;
+      continue;
+    }
     n[hclass]++;
   }
 
-  for(hclass = HC_FIRST; hclass < HC_LAST; hclass++)
-    if (n[hclass] > SaveHist)
-    {
-      mutt_mktemp (tmpfname, sizeof (tmpfname));
-      if ((tmp = safe_fopen (tmpfname, "w+")) == NULL)
-        mutt_perror (tmpfname);
-      break;
-    }
+  if (!regen_file)
+    for(hclass = HC_FIRST; hclass < HC_LAST; hclass++)
+      if (n[hclass] > SaveHist)
+      {
+        regen_file = 1;
+        break;
+      }
 
-  if (tmp != NULL)
+  if (regen_file)
   {
+    mutt_mktemp (tmpfname, sizeof (tmpfname));
+    if ((tmp = safe_fopen (tmpfname, "w+")) == NULL)
+    {
+      mutt_perror (tmpfname);
+      goto cleanup;
+    }
     rewind (f);
     line = 0;
     while ((linebuf = mutt_read_line (linebuf, &buflen, f, &line, 0)) != NULL)
     {
-      if (sscanf (linebuf, "%d", &hclass) < 1 || hclass < 0)
+      if (sscanf (linebuf, "%d:%n", &hclass, &read) < 1 || read == 0 ||
+          *(p = linebuf + strlen (linebuf) - 1) != '|' || hclass < 0)
       {
         mutt_error (_("Bad history file format (line %d)"), line);
         goto cleanup;
       }
-      /* silently ignore too high class (probably newer mutt) */
       if (hclass >= HC_LAST)
 	continue;
+      *p = '\0';
+      if (option (OPTHISTREMOVEDUPS) &&
+          (dup_hash_dec (dup_hashes[hclass], linebuf + read) != 0))
+        continue;
+      *p = '|';
       if (n[hclass]-- <= SaveHist)
         fprintf (tmp, "%s\n", linebuf);
     }
@@ -201,6 +267,9 @@ cleanup:
     safe_fclose (&tmp);
     unlink (tmpfname);
   }
+  if (option (OPTHISTREMOVEDUPS))
+    for (hclass = 0; hclass < HC_LAST; hclass++)
+      hash_destroy (&dup_hashes[hclass], NULL);
 }
 
 static void save_history (history_class_t hclass, const char *s)

init.h

@@ -1083,12 +1083,15 @@ struct option_t MuttVars[] = {
   /*
   ** .pp
   ** The file in which Mutt will save its history.
+  ** .pp
+  ** Also see $$save_history.
   */
   { "history_remove_dups", DT_BOOL, R_NONE, OPTHISTREMOVEDUPS, 0 },
   /*
   ** .pp
   ** When \fIset\fP, all of the string history will be scanned for duplicates
-  ** when a new entry is added.
+  ** when a new entry is added.  Duplicate entries in the $$history_file will
+  ** also be removed when it is periodically compacted.
   */
   { "honor_disposition", DT_BOOL, R_NONE, OPTHONORDISP, 0 },
   /*