mbox.c

Go to the documentation of this file.

 
#include "config.h"
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <stdbool.h>
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <time.h>
#include <unistd.h>
#include <utime.h>
#include "mutt/lib.h"
#include "address/lib.h"
#include "config/lib.h"
#include "email/lib.h"
#include "core/lib.h"
#include "mutt.h"
#include "lib.h"
#include "progress/lib.h"
#include "globals.h"
#include "muttlib.h"
#include "mx.h"

struct MUpdate
{
  bool valid;    
  LOFF_T hdr;    
  LOFF_T body;   
  long lines;    
  LOFF_T length; 
};

static void mbox_adata_free(void **ptr)
{
  if (!ptr || !*ptr)
    return;
 
  struct MboxAccountData *m = *ptr;
 
  mutt_file_fclose(&m->fp);
  FREE(ptr);
}

static struct MboxAccountData *mbox_adata_new(void)
{
  return MUTT_MEM_CALLOC(1, struct MboxAccountData);
}

static int init_mailbox(struct Mailbox *m)
{
  if (!m || !m->account)
    return -1;
  if ((m->type != MUTT_MBOX) && (m->type != MUTT_MMDF))
    return -1;
  if (m->account->adata)
    return 0;
 
  m->account->adata = mbox_adata_new();
  m->account->adata_free = mbox_adata_free;
  return 0;
}

static struct MboxAccountData *mbox_adata_get(struct Mailbox *m)
{
  if (init_mailbox(m) == -1)
    return NULL;
  return m->account->adata;
}

static int mbox_lock_mailbox(struct Mailbox *m, bool excl, bool retry)
{
  struct MboxAccountData *adata = mbox_adata_get(m);
  if (!adata)
    return -1;
 
  int rc = mutt_file_lock(fileno(adata->fp), excl, retry);
  if (rc == 0)
  {
    adata->locked = true;
  }
  else if (retry && !excl)
  {
    m->readonly = true;
    return 0;
  }
 
  return rc;
}

static void mbox_unlock_mailbox(struct Mailbox *m)
{
  struct MboxAccountData *adata = mbox_adata_get(m);
  if (!adata)
    return;
 
  if (adata->locked)
  {
    fflush(adata->fp);
 
    mutt_file_unlock(fileno(adata->fp));
    adata->locked = false;
  }
}

static enum MxOpenReturns mmdf_parse_mailbox(struct Mailbox *m)
{
  if (!m)
    return MX_OPEN_ERROR;
 
  struct MboxAccountData *adata = mbox_adata_get(m);
  if (!adata)
    return MX_OPEN_ERROR;
 
  char buf[8192] = { 0 };
  char return_path[1024] = { 0 };
  int count = 0;
  int lines;
  time_t t = 0;
  LOFF_T loc, tmploc;
  struct Email *e = NULL;
  struct stat st = { 0 };
  struct Progress *progress = NULL;
  enum MxOpenReturns rc = MX_OPEN_ERROR;
 
  /* Record mailbox timestamps and size for change detection */
  if (stat(mailbox_path(m), &st) == -1)
  {
    mutt_perror("%s", mailbox_path(m));
    goto fail;
  }
  mutt_file_get_stat_timespec(&adata->atime, &st, MUTT_STAT_ATIME);
  mutt_file_get_stat_timespec(&adata->mtime, &st, MUTT_STAT_MTIME);
  m->size = st.st_size;
 
  if (m->verbose)
  {
    progress = progress_new(MUTT_PROGRESS_READ, 0);
    progress_set_message(progress, _("Reading %s..."), mailbox_path(m));
  }
 
  /* Read the file line-by-line looking for MMDF separator lines ("\001\001\001\001\n") */
  while (true)
  {
    if (!fgets(buf, sizeof(buf) - 1, adata->fp))
      break;
 
    if (SigInt)
      break;
 
    if (mutt_str_equal(buf, MMDF_SEP))
    {
      loc = ftello(adata->fp);
      if (loc < 0)
        goto fail;
 
      count++;
      progress_update(progress, count, (int) (loc / (m->size / 100 + 1)));
 
      /* Allocate a new Email for this message */
      mx_alloc_memory(m, m->msg_count);
      e = email_new();
      m->emails[m->msg_count] = e;
      e->offset = loc;
      e->index = m->msg_count;
 
      if (!fgets(buf, sizeof(buf) - 1, adata->fp))
      {
        mutt_debug(LL_DEBUG1, "unexpected EOF\n");
        email_free(&m->emails[m->msg_count]);
        break;
      }
 
      /* Try to parse "From " line for return path and date */
      return_path[0] = '\0';
 
      if (!is_from(buf, return_path, sizeof(return_path), &t))
      {
        if (!mutt_file_seek(adata->fp, loc, SEEK_SET))
        {
          mutt_error(_("Mailbox is corrupt"));
          goto fail;
        }
      }
      else
      {
        e->received = t - mutt_date_local_tz(t);
      }
 
      /* Parse RFC 822 headers to populate the envelope */
      e->env = mutt_rfc822_read_header(adata->fp, e, false, false);
 
      loc = ftello(adata->fp);
      if (loc < 0)
        goto fail;
 
      /* Try to validate the Content-Length by seeking to the expected end
       * and verifying an MMDF separator is there */
      if ((e->body->length > 0) && (e->lines > 0))
      {
        tmploc = loc + e->body->length;
 
        if ((tmploc > 0) && (tmploc < m->size))
        {
          if (!mutt_file_seek(adata->fp, tmploc, SEEK_SET) ||
              !fgets(buf, sizeof(buf) - 1, adata->fp) || !mutt_str_equal(MMDF_SEP, buf))
          {
            (void) mutt_file_seek(adata->fp, loc, SEEK_SET);
            e->body->length = -1;
          }
        }
        else
        {
          e->body->length = -1;
        }
      }
      else
      {
        e->body->length = -1;
      }
 
      /* If Content-Length was missing or invalid, scan for the next separator */
      if (e->body->length < 0)
      {
        lines = -1;
        do
        {
          loc = ftello(adata->fp);
          if (loc < 0)
            goto fail;
          if (!fgets(buf, sizeof(buf) - 1, adata->fp))
            break;
          lines++;
        } while (!mutt_str_equal(buf, MMDF_SEP));
 
        e->lines = lines;
        e->body->length = loc - e->body->offset;
      }
 
      /* Populate return path and from address if missing from headers */
      if (TAILQ_EMPTY(&e->env->return_path) && return_path[0])
        mutt_addrlist_parse(&e->env->return_path, return_path);
 
      if (TAILQ_EMPTY(&e->env->from))
        mutt_addrlist_copy(&e->env->from, &e->env->return_path, false);
 
      m->msg_count++;
    }
    else
    {
      mutt_debug(LL_DEBUG1, "corrupt mailbox\n");
      mutt_error(_("Mailbox is corrupt"));
      goto fail;
    }
  }
 
  if (SigInt)
  {
    SigInt = false;
    rc = MX_OPEN_ABORT; /* action aborted */
    goto fail;
  }
 
  rc = MX_OPEN_OK;
fail:
  progress_free(&progress);
  return rc;
}

static enum MxOpenReturns mbox_parse_mailbox(struct Mailbox *m)
{
  if (!m)
    return MX_OPEN_ERROR;
 
  struct MboxAccountData *adata = mbox_adata_get(m);
  if (!adata)
    return MX_OPEN_ERROR;
 
  struct stat st = { 0 };
  char buf[8192] = { 0 };
  char return_path[1024] = { 0 };
  struct Email *e_cur = NULL;
  time_t t = 0;
  int count = 0, lines = 0;
  bool has_mbox_sep = false;
  bool expect_from_line = true;
  LOFF_T loc;
  struct Progress *progress = NULL;
  enum MxOpenReturns rc = MX_OPEN_ERROR;
 
  /* Save information about the folder at the time we opened it. */
  if (stat(mailbox_path(m), &st) == -1)
  {
    mutt_perror("%s", mailbox_path(m));
    goto fail;
  }
 
  m->size = st.st_size;
  mutt_file_get_stat_timespec(&adata->mtime, &st, MUTT_STAT_MTIME);
  mutt_file_get_stat_timespec(&adata->atime, &st, MUTT_STAT_ATIME);
 
  if (!m->readonly)
    m->readonly = access(mailbox_path(m), W_OK) ? true : false;
 
  if (m->verbose)
  {
    progress = progress_new(MUTT_PROGRESS_READ, 0);
    progress_set_message(progress, _("Reading %s..."), mailbox_path(m));
  }
 
  loc = ftello(adata->fp);
  if (loc < 0)
  {
    mutt_debug(LL_DEBUG1, "ftello: %s (errno %d)\n", strerror(errno), errno);
    loc = 0;
  }
 
  while ((fgets(buf, sizeof(buf), adata->fp)) && !SigInt)
  {
    if (is_from(buf, return_path, sizeof(return_path), &t))
    {
      /* Save the Content-Length of the previous message */
      if (count > 0)
      {
        struct Email *e = m->emails[m->msg_count - 1];
        if (!has_mbox_sep)
        {
          mutt_debug(LL_DEBUG1, "mbox_parse_mailbox: missing separator at location: " OFF_T_FMT "\n",
                     loc);
        }
        if (e->body->length < 0)
        {
          e->body->length = loc - e->body->offset - (has_mbox_sep ? 1 : 0);
          if (e->body->length < 0)
            e->body->length = 0;
        }
        if (e->lines == 0)
          e->lines = lines ? lines - 1 : 0;
      }
 
      count++;
      expect_from_line = false;
 
      progress_update(progress, count, (int) (ftello(adata->fp) / (m->size / 100 + 1)));
 
      mx_alloc_memory(m, m->msg_count);
 
      m->emails[m->msg_count] = email_new();
      e_cur = m->emails[m->msg_count];
      e_cur->received = t - mutt_date_local_tz(t);
      e_cur->offset = loc;
      e_cur->index = m->msg_count;
 
      e_cur->env = mutt_rfc822_read_header(adata->fp, e_cur, false, false);
 
      /* if we know how long this message is, either just skip over the body,
       * or if we don't know how many lines there are, count them now (this will
       * save time by not having to search for the next message marker).  */
      if (e_cur->body->length > 0)
      {
        LOFF_T tmploc;
 
        loc = ftello(adata->fp);
        if (loc < 0)
        {
          mutt_debug(LL_DEBUG1, "ftello: %s (errno %d)\n", strerror(errno), errno);
          loc = 0;
        }
 
        /* The test below avoids a potential integer overflow if the
         * content-length is huge (thus necessarily invalid).  */
        tmploc = (e_cur->body->length < m->size) ? (loc + e_cur->body->length + 1) : -1;
 
        if ((tmploc > 0) && (tmploc < m->size))
        {
          /* check to see if the content-length looks valid.  we expect to
           * to see a valid message separator at this point in the stream */
          if (!mutt_file_seek(adata->fp, tmploc, SEEK_SET) ||
              !fgets(buf, sizeof(buf), adata->fp) || !mutt_str_startswith(buf, "From "))
          {
            mutt_debug(LL_DEBUG1, "bad content-length in message %d (cl=" OFF_T_FMT ")\n",
                       e_cur->index, e_cur->body->length);
            mutt_debug(LL_DEBUG1, "    LINE: %s", buf);
            /* nope, return the previous position */
            (void) mutt_file_seek(adata->fp, loc, SEEK_SET);
            e_cur->body->length = -1;
          }
        }
        else if (tmploc != m->size)
        {
          /* content-length would put us past the end of the file, so it
           * must be wrong */
          e_cur->body->length = -1;
        }
 
        if (e_cur->body->length != -1)
        {
          /* good content-length.  check to see if we know how many lines
           * are in this message.  */
          if (e_cur->lines == 0)
          {
            LOFF_T cl = e_cur->body->length;
 
            /* count the number of lines in this message */
            (void) mutt_file_seek(adata->fp, loc, SEEK_SET);
            while (cl-- > 0)
            {
              if (fgetc(adata->fp) == '\n')
                e_cur->lines++;
            }
          }
 
          /* return to the offset of the next *mbox* separator */
          (void) mutt_file_seek(adata->fp, tmploc - 1, SEEK_SET);
          expect_from_line = true;
        }
      }
 
      m->msg_count++;
 
      if (TAILQ_EMPTY(&e_cur->env->return_path) && return_path[0])
      {
        mutt_addrlist_parse(&e_cur->env->return_path, return_path);
      }
 
      if (TAILQ_EMPTY(&e_cur->env->from))
        mutt_addrlist_copy(&e_cur->env->from, &e_cur->env->return_path, false);
 
      lines = 0;
      has_mbox_sep = false;
    }
    else
    {
      lines++;
      has_mbox_sep = mutt_str_equal(MBOX_SEP, buf);
      if (expect_from_line && !has_mbox_sep)
      {
        mutt_debug(LL_DEBUG1, "mbox_parse_mailbox: missing From_ line at location: " OFF_T_FMT "\n",
                   loc);
        mutt_error(_("Mailbox is corrupt"));
        goto fail;
      }
    }
 
    loc = ftello(adata->fp);
  }
 
  /* Only set the content-length of the previous message if we have read more
   * than one message during _this_ invocation.  If this routine is called
   * when new mail is received, we need to make sure not to clobber what
   * previously was the last message since the headers may be sorted.  */
  if (count > 0)
  {
    struct Email *e = m->emails[m->msg_count - 1];
    if (!has_mbox_sep)
    {
      mutt_debug(LL_DEBUG1,
                 "mbox_parse_mailbox: missing separator at location: " OFF_T_FMT "\n", loc);
    }
    if (e->body->length < 0)
    {
      e->body->length = ftello(adata->fp) - e->body->offset - (has_mbox_sep ? 1 : 0);
      if (e->body->length < 0)
        e->body->length = 0;
    }
 
    if (e->lines == 0)
      e->lines = lines ? lines - 1 : 0;
  }
 
  if (SigInt)
  {
    SigInt = false;
    rc = MX_OPEN_ABORT;
    goto fail; /* action aborted */
  }
 
  rc = MX_OPEN_OK;
fail:
  progress_free(&progress);
  return rc;
}

static int reopen_mailbox(struct Mailbox *m)
{
  if (!m)
    return -1;
 
  struct MboxAccountData *adata = mbox_adata_get(m);
  if (!adata)
    return -1;
 
  bool (*cmp_headers)(const struct Email *, const struct Email *) = NULL;
  struct Email **e_old = NULL;
  int old_msg_count;
  bool msg_mod = false;
  int rc = -1;
 
  /* silent operations */
  m->verbose = false;
 
  /* our heuristics require the old mailbox to be unsorted */
  const enum EmailSortType c_sort = cs_subset_sort(NeoMutt->sub, "sort");
  if (c_sort != EMAIL_SORT_UNSORTED)
  {
    cs_subset_str_native_set(NeoMutt->sub, "sort", EMAIL_SORT_UNSORTED, NULL);
    mailbox_changed(m, NT_MAILBOX_RESORT);
    cs_subset_str_native_set(NeoMutt->sub, "sort", c_sort, NULL);
  }
 
  e_old = NULL;
  old_msg_count = 0;
 
  /* simulate a close */
  mutt_hash_free(&m->id_hash);
  mutt_hash_free(&m->subj_hash);
  mutt_hash_free(&m->label_hash);
  FREE(&m->v2r);
  if (m->readonly)
  {
    for (int i = 0; i < m->msg_count; i++)
      email_free(&(m->emails[i])); /* nothing to do! */
    FREE(&m->emails);
  }
  else
  {
    /* save the old headers */
    old_msg_count = m->msg_count;
    e_old = m->emails;
    m->emails = NULL;
  }
 
  m->email_max = 0; /* force allocation of new headers */
  m->msg_count = 0;
  m->vcount = 0;
  m->msg_tagged = 0;
  m->msg_deleted = 0;
  m->msg_new = 0;
  m->msg_unread = 0;
  m->msg_flagged = 0;
  m->changed = false;
  m->id_hash = NULL;
  m->subj_hash = NULL;
  mutt_make_label_hash(m);
 
  switch (m->type)
  {
    case MUTT_MBOX:
    case MUTT_MMDF:
      cmp_headers = email_cmp_strict;
      mutt_file_fclose(&adata->fp);
      adata->fp = mutt_file_fopen(mailbox_path(m), "r");
      if (!adata->fp)
        rc = -1;
      else if (m->type == MUTT_MBOX)
        rc = mbox_parse_mailbox(m);
      else
        rc = mmdf_parse_mailbox(m);
      break;
 
    default:
      rc = -1;
      break;
  }
 
  if (rc == -1)
  {
    /* free the old headers */
    for (int i = 0; i < old_msg_count; i++)
      email_free(&(e_old[i]));
    FREE(&e_old);
 
    m->verbose = true;
    return -1;
  }
 
  mutt_file_touch_atime(fileno(adata->fp));
 
  /* now try to recover the old flags */
 
  if (!m->readonly)
  {
    for (int i = 0; i < m->msg_count; i++)
    {
      bool found = false;
 
      /* some messages have been deleted, and new  messages have been
       * appended at the end; the heuristic is that old messages have then
       * "advanced" towards the beginning of the folder, so we begin the
       * search at index "i" */
      int j;
      for (j = i; j < old_msg_count; j++)
      {
        if (!e_old[j])
          continue;
        if (cmp_headers(m->emails[i], e_old[j]))
        {
          found = true;
          break;
        }
      }
      if (!found)
      {
        for (j = 0; (j < i) && (j < old_msg_count); j++)
        {
          if (!e_old[j])
            continue;
          if (cmp_headers(m->emails[i], e_old[j]))
          {
            found = true;
            break;
          }
        }
      }
 
      if (found)
      {
        m->changed = true;
        if (e_old[j]->changed)
        {
          /* Only update the flags if the old header was changed;
           * otherwise, the header may have been modified externally,
           * and we don't want to lose _those_ changes */
          mutt_set_flag(m, m->emails[i], MUTT_FLAG, e_old[j]->flagged, true);
          mutt_set_flag(m, m->emails[i], MUTT_REPLIED, e_old[j]->replied, true);
          mutt_set_flag(m, m->emails[i], MUTT_OLD, e_old[j]->old, true);
          mutt_set_flag(m, m->emails[i], MUTT_READ, e_old[j]->read, true);
        }
        mutt_set_flag(m, m->emails[i], MUTT_DELETE, e_old[j]->deleted, true);
        mutt_set_flag(m, m->emails[i], MUTT_PURGE, e_old[j]->purge, true);
        mutt_set_flag(m, m->emails[i], MUTT_TAG, e_old[j]->tagged, true);
 
        /* we don't need this header any more */
        email_free(&(e_old[j]));
      }
    }
 
    /* free the remaining old emails */
    for (int j = 0; j < old_msg_count; j++)
    {
      if (e_old[j])
      {
        email_free(&(e_old[j]));
        msg_mod = true;
      }
    }
    FREE(&e_old);
  }
 
  mailbox_changed(m, NT_MAILBOX_UPDATE);
  m->verbose = true;
 
  return (m->changed || msg_mod) ? MX_STATUS_REOPENED : MX_STATUS_NEW_MAIL;
}

static bool mbox_has_new(struct Mailbox *m)
{
  for (int i = 0; i < m->msg_count; i++)
  {
    struct Email *e = m->emails[i];
    if (!e)
      break;
    if (!e->deleted && !e->read && !e->old)
      return true;
  }
  return false;
}

void mbox_reset_atime(struct Mailbox *m, struct stat *st)
{
  struct stat st2 = { 0 };
  if (!st)
  {
    if (stat(mailbox_path(m), &st2) < 0)
      return;
    st = &st2;
  }
 
  struct utimbuf utimebuf = { 0 };
  utimebuf.actime = st->st_atime;
  utimebuf.modtime = st->st_mtime;
 
  /* When $mbox_check_recent is set, existing new mail is ignored, so do not
   * reset the atime to mtime-1 to signal new mail.  */
  const bool c_mail_check_recent = cs_subset_bool(NeoMutt->sub, "mail_check_recent");
  if (!c_mail_check_recent && (utimebuf.actime >= utimebuf.modtime) && mbox_has_new(m))
  {
    utimebuf.actime = utimebuf.modtime - 1;
  }
 
  utime(mailbox_path(m), &utimebuf);
}

static bool mbox_ac_owns_path(struct Account *a, const char *path)
{
  if ((a->type != MUTT_MBOX) && (a->type != MUTT_MMDF))
    return false;
 
  struct Mailbox **mp = ARRAY_FIRST(&a->mailboxes);
  if (!mp)
    return false;
 
  return mutt_str_equal(mailbox_path(*mp), path);
}

static bool mbox_ac_add(struct Account *a, struct Mailbox *m)
{
  return true;
}

static FILE *mbox_open_readwrite(struct Mailbox *m)
{
  FILE *fp = mutt_file_fopen(mailbox_path(m), "r+");
  if (fp)
    m->readonly = false;
  return fp;
}

static FILE *mbox_open_readonly(struct Mailbox *m)
{
  FILE *fp = mutt_file_fopen(mailbox_path(m), "r");
  if (fp)
    m->readonly = true;
  return fp;
}

static enum MxOpenReturns mbox_mbox_open(struct Mailbox *m)
{
  if (init_mailbox(m) != 0)
    return MX_OPEN_ERROR;
 
  struct MboxAccountData *adata = mbox_adata_get(m);
  if (!adata)
    return MX_OPEN_ERROR;
 
  adata->fp = m->readonly ? NULL : mbox_open_readwrite(m);
  if (!adata->fp)
  {
    adata->fp = mbox_open_readonly(m);
  }
  if (!adata->fp)
  {
    mutt_perror("%s", mailbox_path(m));
    return MX_OPEN_ERROR;
  }
 
  mutt_sig_block();
  if (mbox_lock_mailbox(m, false, true) == -1)
  {
    mutt_sig_unblock();
    return MX_OPEN_ERROR;
  }
 
  m->has_new = true;
  enum MxOpenReturns rc = MX_OPEN_ERROR;
  if (m->type == MUTT_MBOX)
    rc = mbox_parse_mailbox(m);
  else if (m->type == MUTT_MMDF)
    rc = mmdf_parse_mailbox(m);
  else
    rc = MX_OPEN_ERROR;
 
  if (!mbox_has_new(m))
    m->has_new = false;
  clearerr(adata->fp); // Clear the EOF flag
  mutt_file_touch_atime(fileno(adata->fp));
 
  mbox_unlock_mailbox(m);
  mutt_sig_unblock();
  return rc;
}

static bool mbox_mbox_open_append(struct Mailbox *m, OpenMailboxFlags flags)
{
  if (init_mailbox(m) != 0)
    return false;
 
  struct MboxAccountData *adata = mbox_adata_get(m);
  if (!adata)
    return false;
 
  if (!adata->fp)
  {
    // create dir recursively
    char *tmp_path = mutt_path_dirname(mailbox_path(m));
    if (mutt_file_mkdir(tmp_path, S_IRWXU) == -1)
    {
      mutt_perror("%s", mailbox_path(m));
      FREE(&tmp_path);
      return false;
    }
    FREE(&tmp_path);
 
    adata->fp = mutt_file_fopen(mailbox_path(m), "a+");
    if (!adata->fp)
    {
      mutt_perror("%s", mailbox_path(m));
      return false;
    }
 
    if (mbox_lock_mailbox(m, true, true) != 0)
    {
      mutt_error(_("Couldn't lock %s"), mailbox_path(m));
      mutt_file_fclose(&adata->fp);
      return false;
    }
  }
 
  if (!mutt_file_seek(adata->fp, 0, SEEK_END))
  {
    mutt_file_fclose(&adata->fp);
    return false;
  }
 
  return true;
}

static enum MxStatus mbox_mbox_check(struct Mailbox *m)
{
  struct MboxAccountData *adata = mbox_adata_get(m);
  if (!adata)
    return MX_STATUS_ERROR;
 
  if (!adata->fp)
  {
    if (mbox_mbox_open(m) != MX_OPEN_OK)
      return MX_STATUS_ERROR;
    mailbox_changed(m, NT_MAILBOX_INVALID);
  }
  if (!adata->fp)
    return MX_STATUS_ERROR;
 
  struct stat st = { 0 };
  bool unlock = false;
  bool modified = false;
 
  if (stat(mailbox_path(m), &st) == 0)
  {
    if ((mutt_file_stat_timespec_compare(&st, MUTT_STAT_MTIME, &adata->mtime) == 0) &&
        (st.st_size == m->size))
    {
      return MX_STATUS_OK;
    }
 
    if (st.st_size == m->size)
    {
      /* the file was touched, but it is still the same length, so just exit */
      mutt_file_get_stat_timespec(&adata->mtime, &st, MUTT_STAT_MTIME);
      return MX_STATUS_OK;
    }
 
    if (st.st_size > m->size)
    {
      /* lock the file if it isn't already */
      if (!adata->locked)
      {
        mutt_sig_block();
        if (mbox_lock_mailbox(m, false, false) == -1)
        {
          mutt_sig_unblock();
          /* we couldn't lock the mailbox, but nothing serious happened:
           * probably the new mail arrived: no reason to wait till we can
           * parse it: we'll get it on the next pass */
          return MX_STATUS_LOCKED;
        }
        unlock = 1;
      }
 
      /* Check to make sure that the only change to the mailbox is that
       * message(s) were appended to this file.  My heuristic is that we should
       * see the message separator at *exactly* what used to be the end of the
       * folder.  */
      char buf[1024] = { 0 };
      if (!mutt_file_seek(adata->fp, m->size, SEEK_SET))
      {
        goto error;
      }
      if (fgets(buf, sizeof(buf), adata->fp))
      {
        if (((m->type == MUTT_MBOX) && mutt_str_startswith(buf, "From ")) ||
            ((m->type == MUTT_MMDF) && mutt_str_equal(buf, MMDF_SEP)))
        {
          if (!mutt_file_seek(adata->fp, m->size, SEEK_SET))
          {
            goto error;
          }
 
          int old_msg_count = m->msg_count;
          if (m->type == MUTT_MBOX)
            mbox_parse_mailbox(m);
          else
            mmdf_parse_mailbox(m);
 
          if (m->msg_count > old_msg_count)
            mailbox_changed(m, NT_MAILBOX_INVALID);
 
          /* Only unlock the folder if it was locked inside of this routine.
           * It may have been locked elsewhere, like in
           * mutt_checkpoint_mailbox().  */
          if (unlock)
          {
            mbox_unlock_mailbox(m);
            mutt_sig_unblock();
          }
 
          return MX_STATUS_NEW_MAIL; /* signal that new mail arrived */
        }
        else
        {
          modified = true;
        }
      }
      else
      {
        mutt_debug(LL_DEBUG1, "fgets returned NULL\n");
        modified = true;
      }
    }
    else
    {
      modified = true;
    }
  }
 
  if (modified)
  {
    if (reopen_mailbox(m) != -1)
    {
      mailbox_changed(m, NT_MAILBOX_INVALID);
      if (unlock)
      {
        mbox_unlock_mailbox(m);
        mutt_sig_unblock();
      }
      return MX_STATUS_REOPENED;
    }
  }
 
  /* fatal error */
 
error:
  mbox_unlock_mailbox(m);
  mx_fastclose_mailbox(m, false);
  mutt_sig_unblock();
  mutt_error(_("Mailbox was corrupted"));
  return MX_STATUS_ERROR;
}

static enum MxStatus mbox_mbox_sync(struct Mailbox *m)
{
  struct MboxAccountData *adata = mbox_adata_get(m);
  if (!adata)
    return MX_STATUS_ERROR;
 
  struct Buffer *tempfile = NULL;
  char buf[32] = { 0 };
  int j;
  bool unlink_tempfile = false;
  bool need_sort = false; /* flag to resort mailbox if new mail arrives */
  int first = -1;         /* first message to be written */
  LOFF_T offset;          /* location in mailbox to write changed messages */
  struct stat st = { 0 };
  struct MUpdate *new_offset = NULL;
  struct MUpdate *old_offset = NULL;
  FILE *fp = NULL;
  struct Progress *progress = NULL;
  enum MxStatus rc = MX_STATUS_ERROR;
 
  /* sort message by their position in the mailbox on disk */
  const enum EmailSortType c_sort = cs_subset_sort(NeoMutt->sub, "sort");
  if (c_sort != EMAIL_SORT_UNSORTED)
  {
    mutt_sort_unsorted(m);
    need_sort = true;
  }
 
  /* need to open the file for writing in such a way that it does not truncate
   * the file, so use read-write mode.  */
  adata->fp = freopen(mailbox_path(m), "r+", adata->fp);
  if (!adata->fp)
  {
    mx_fastclose_mailbox(m, false);
    mutt_error(_("Fatal error!  Could not reopen mailbox!"));
    goto fatal;
  }
 
  mutt_sig_block();
 
  if (mbox_lock_mailbox(m, true, true) == -1)
  {
    mutt_sig_unblock();
    mutt_error(_("Unable to lock mailbox"));
    goto bail;
  }
 
  /* Check to make sure that the file hasn't changed on disk */
  enum MxStatus check = mbox_mbox_check(m);
  if ((check == MX_STATUS_NEW_MAIL) || (check == MX_STATUS_REOPENED))
  {
    /* new mail arrived, or mailbox reopened */
    rc = check;
    goto bail;
  }
  else if (check < 0)
  {
    goto fatal;
  }
 
  /* Create a temporary file to write the new version of the mailbox in. */
  tempfile = buf_pool_get();
  buf_mktemp(tempfile);
  int fd = open(buf_string(tempfile), O_WRONLY | O_EXCL | O_CREAT, 0600);
  if ((fd == -1) || !(fp = fdopen(fd, "w")))
  {
    if (fd != -1)
    {
      close(fd);
      unlink_tempfile = true;
    }
    mutt_error(_("Could not create temporary file"));
    goto bail;
  }
  unlink_tempfile = true;
 
  /* find the first deleted/changed message.  we save a lot of time by only
   * rewriting the mailbox from the point where it has actually changed.  */
  int i = 0;
  for (; (i < m->msg_count) && !m->emails[i]->deleted &&
         !m->emails[i]->changed && !m->emails[i]->attach_del;
       i++)
  {
  }
  if (i == m->msg_count)
  {
    /* this means m->changed or m->msg_deleted was set, but no
     * messages were found to be changed or deleted.  This should
     * never happen, is we presume it is a bug in neomutt.  */
    mutt_error(_("sync: mbox modified, but no modified messages (report this bug)"));
    mutt_debug(LL_DEBUG1, "no modified messages\n");
    goto bail;
  }
 
  /* save the index of the first changed/deleted message */
  first = i;
  /* where to start overwriting */
  offset = m->emails[i]->offset;
 
  /* the offset stored in the header does not include the MMDF_SEP, so make
   * sure we seek to the correct location */
  if (m->type == MUTT_MMDF)
    offset -= (sizeof(MMDF_SEP) - 1);
 
  /* allocate space for the new offsets */
  new_offset = MUTT_MEM_CALLOC(m->msg_count - first, struct MUpdate);
  old_offset = MUTT_MEM_CALLOC(m->msg_count - first, struct MUpdate);
 
  if (m->verbose)
  {
    progress = progress_new(MUTT_PROGRESS_WRITE, m->msg_count);
    progress_set_message(progress, _("Writing %s..."), mailbox_path(m));
  }
 
  for (i = first, j = 0; i < m->msg_count; i++)
  {
    progress_update(progress, i, i / (m->msg_count / 100 + 1));
    /* back up some information which is needed to restore offsets when
     * something fails.  */
 
    old_offset[i - first].valid = true;
    old_offset[i - first].hdr = m->emails[i]->offset;
    old_offset[i - first].body = m->emails[i]->body->offset;
    old_offset[i - first].lines = m->emails[i]->lines;
    old_offset[i - first].length = m->emails[i]->body->length;
 
    if (!m->emails[i]->deleted)
    {
      j++;
 
      if (m->type == MUTT_MMDF)
      {
        if (fputs(MMDF_SEP, fp) == EOF)
        {
          mutt_perror("%s", buf_string(tempfile));
          goto bail;
        }
      }
 
      /* save the new offset for this message.  we add 'offset' because the
       * temporary file only contains saved message which are located after
       * 'offset' in the real mailbox */
      new_offset[i - first].hdr = ftello(fp) + offset;
 
      struct Message *msg = mx_msg_open(m, m->emails[i]);
      const int rc2 = mutt_copy_message(fp, m->emails[i], msg, MUTT_CM_UPDATE,
                                        CH_FROM | CH_UPDATE | CH_UPDATE_LEN, 0);
      mx_msg_close(m, &msg);
      if (rc2 != 0)
      {
        mutt_perror("%s", buf_string(tempfile));
        goto bail;
      }
 
      /* Since messages could have been deleted, the offsets stored in memory
       * will be wrong, so update what we can, which is the offset of this
       * message, and the offset of the body.  If this is a multipart message,
       * we just flush the in memory cache so that the message will be reparsed
       * if the user accesses it later.  */
      new_offset[i - first].body = ftello(fp) - m->emails[i]->body->length + offset;
      mutt_body_free(&m->emails[i]->body->parts);
 
      if (m->type == MUTT_MMDF)
      {
        if (fputs(MMDF_SEP, fp) == EOF)
        {
          mutt_perror("%s", buf_string(tempfile));
          goto bail;
        }
      }
      else
      {
        if (fputs(MBOX_SEP, fp) == EOF)
        {
          mutt_perror("%s", buf_string(tempfile));
          goto bail;
        }
      }
    }
  }
 
  if (mutt_file_fclose(&fp) != 0)
  {
    mutt_debug(LL_DEBUG1, "mutt_file_fclose (&) returned non-zero\n");
    mutt_perror("%s", buf_string(tempfile));
    goto bail;
  }
 
  /* Save the state of this folder. */
  if (stat(mailbox_path(m), &st) == -1)
  {
    mutt_perror("%s", mailbox_path(m));
    goto bail;
  }
 
  unlink_tempfile = false;
 
  fp = mutt_file_fopen(buf_string(tempfile), "r");
  if (!fp)
  {
    mutt_sig_unblock();
    mx_fastclose_mailbox(m, false);
    mutt_debug(LL_DEBUG1, "unable to reopen temp copy of mailbox!\n");
    mutt_perror("%s", buf_string(tempfile));
    FREE(&new_offset);
    FREE(&old_offset);
    goto fatal;
  }
 
  if (!mutt_file_seek(adata->fp, offset, SEEK_SET) || /* seek the append location */
      /* do a sanity check to make sure the mailbox looks ok */
      !fgets(buf, sizeof(buf), adata->fp) ||
      ((m->type == MUTT_MBOX) && !mutt_str_startswith(buf, "From ")) ||
      ((m->type == MUTT_MMDF) && !mutt_str_equal(MMDF_SEP, buf)))
  {
    mutt_debug(LL_DEBUG1, "message not in expected position\n");
    mutt_debug(LL_DEBUG1, "    LINE: %s\n", buf);
    i = -1;
  }
  else
  {
    if (!mutt_file_seek(adata->fp, offset, SEEK_SET)) /* return to proper offset */
    {
      i = -1;
    }
    else
    {
      /* copy the temp mailbox back into place starting at the first
       * change/deleted message */
      if (m->verbose)
        mutt_message(_("Committing changes..."));
      i = mutt_file_copy_stream(fp, adata->fp);
 
      if (ferror(adata->fp))
        i = -1;
    }
    if (i >= 0)
    {
      m->size = ftello(adata->fp); /* update the mailbox->size of the mailbox */
      if ((m->size < 0) || (ftruncate(fileno(adata->fp), m->size) != 0))
      {
        i = -1;
        mutt_debug(LL_DEBUG1, "ftruncate() failed\n");
      }
    }
  }
 
  mutt_file_fclose(&fp);
  mbox_unlock_mailbox(m);
 
  if ((mutt_file_fclose(&adata->fp) != 0) || (i == -1))
  {
    /* error occurred while writing the mailbox back, so keep the temp copy around */
 
    struct Buffer *savefile = buf_pool_get();
 
    const char *const c_tmp_dir = cs_subset_path(NeoMutt->sub, "tmp_dir");
    buf_printf(savefile, "%s/neomutt.%s-%s-%u", NONULL(c_tmp_dir),
               NONULL(NeoMutt->username), NONULL(ShortHostname), (unsigned int) getpid());
    rename(buf_string(tempfile), buf_string(savefile));
    mutt_sig_unblock();
    mx_fastclose_mailbox(m, false);
    pretty_mailbox(savefile);
    mutt_error(_("Write failed!  Saved partial mailbox to %s"), buf_string(savefile));
    buf_pool_release(&savefile);
    FREE(&new_offset);
    FREE(&old_offset);
    goto fatal;
  }
 
  /* Restore the previous access/modification times */
  mbox_reset_atime(m, &st);
 
  /* reopen the mailbox in read-only mode */
  adata->fp = mbox_open_readwrite(m);
  if (!adata->fp)
  {
    adata->fp = mbox_open_readonly(m);
  }
  if (!adata->fp)
  {
    unlink(buf_string(tempfile));
    mutt_sig_unblock();
    mx_fastclose_mailbox(m, false);
    mutt_error(_("Fatal error!  Could not reopen mailbox!"));
    FREE(&new_offset);
    FREE(&old_offset);
    goto fatal;
  }
 
  /* update the offsets of the rewritten messages */
  for (i = first, j = first; i < m->msg_count; i++)
  {
    if (!m->emails[i]->deleted)
    {
      m->emails[i]->offset = new_offset[i - first].hdr;
      m->emails[i]->body->hdr_offset = new_offset[i - first].hdr;
      m->emails[i]->body->offset = new_offset[i - first].body;
      m->emails[i]->index = j++;
    }
  }
  FREE(&new_offset);
  FREE(&old_offset);
  unlink(buf_string(tempfile)); /* remove partial copy of the mailbox */
  buf_pool_release(&tempfile);
  mutt_sig_unblock();
 
  const bool c_check_mbox_size = cs_subset_bool(NeoMutt->sub, "check_mbox_size");
  if (c_check_mbox_size)
  {
    struct Mailbox *m_tmp = mailbox_find(mailbox_path(m));
    if (m_tmp && !m_tmp->has_new)
      mailbox_update(m_tmp);
  }
 
  progress_free(&progress);
  return 0; /* signal success */
 
bail: /* Come here in case of disaster */
 
  mutt_file_fclose(&fp);
 
  if (tempfile && unlink_tempfile)
    unlink(buf_string(tempfile));
 
  /* restore offsets, as far as they are valid */
  if ((first >= 0) && old_offset)
  {
    for (i = first; (i < m->msg_count) && old_offset[i - first].valid; i++)
    {
      m->emails[i]->offset = old_offset[i - first].hdr;
      m->emails[i]->body->hdr_offset = old_offset[i - first].hdr;
      m->emails[i]->body->offset = old_offset[i - first].body;
      m->emails[i]->lines = old_offset[i - first].lines;
      m->emails[i]->body->length = old_offset[i - first].length;
    }
  }
 
  /* this is ok to call even if we haven't locked anything */
  mbox_unlock_mailbox(m);
 
  mutt_sig_unblock();
  FREE(&new_offset);
  FREE(&old_offset);
 
  adata->fp = freopen(mailbox_path(m), "r", adata->fp);
  if (!adata->fp)
  {
    mutt_error(_("Could not reopen mailbox"));
    mx_fastclose_mailbox(m, false);
    goto fatal;
  }
 
  mailbox_changed(m, NT_MAILBOX_UPDATE);
  if (need_sort)
  {
    /* if the mailbox was reopened, the thread tree will be invalid so make
     * sure to start threading from scratch.  */
    mailbox_changed(m, NT_MAILBOX_RESORT);
  }
 
fatal:
  buf_pool_release(&tempfile);
  progress_free(&progress);
  return rc;
}

static enum MxStatus mbox_mbox_close(struct Mailbox *m)
{
  struct MboxAccountData *adata = mbox_adata_get(m);
  if (!adata)
    return MX_STATUS_ERROR;
 
  if (!adata->fp)
    return MX_STATUS_OK;
 
  if (adata->append)
  {
    mutt_file_unlock(fileno(adata->fp));
    mutt_sig_unblock();
  }
 
  mutt_file_fclose(&adata->fp);
 
  /* fix up the times so mailbox won't get confused */
  if (m->peekonly && !buf_is_empty(&m->pathbuf) &&
      (mutt_file_timespec_compare(&adata->mtime, &adata->atime) > 0))
  {
#ifdef HAVE_UTIMENSAT
    struct timespec ts[2] = { { 0 }, { 0 } };
    ts[0] = adata->atime;
    ts[1] = adata->mtime;
    utimensat(AT_FDCWD, mailbox_path(m), ts, 0);
#else
    struct utimbuf ut = { 0 };
    ut.actime = adata->atime.tv_sec;
    ut.modtime = adata->mtime.tv_sec;
    utime(mailbox_path(m), &ut);
#endif
  }
 
  return MX_STATUS_OK;
}

static bool mbox_msg_open(struct Mailbox *m, struct Message *msg, struct Email *e)
{
  struct MboxAccountData *adata = mbox_adata_get(m);
  if (!adata)
    return false;
 
  msg->fp = mutt_file_fopen(mailbox_path(m), "r");
  if (!msg->fp)
    return false;
 
  return true;
}

static bool mbox_msg_open_new(struct Mailbox *m, struct Message *msg, const struct Email *e)
{
  struct MboxAccountData *adata = mbox_adata_get(m);
  if (!adata)
    return false;
 
  msg->fp = adata->fp;
  return true;
}

static int mbox_msg_commit(struct Mailbox *m, struct Message *msg)
{
  if (fputs(MBOX_SEP, msg->fp) == EOF)
    return -1;
 
  if ((fflush(msg->fp) == EOF) || (fsync(fileno(msg->fp)) == -1))
  {
    mutt_perror(_("Can't write message"));
    return -1;
  }
 
  return 0;
}

static int mbox_msg_close(struct Mailbox *m, struct Message *msg)
{
  if (msg->write)
    msg->fp = NULL;
  else
    mutt_file_fclose(&msg->fp);
 
  return 0;
}

static int mbox_msg_padding_size(struct Mailbox *m)
{
  return 1;
}

enum MailboxType mbox_path_probe(const char *path, const struct stat *st)
{
  if (!st)
    return MUTT_UNKNOWN;
 
  if (S_ISDIR(st->st_mode))
    return MUTT_UNKNOWN;
 
  if (st->st_size == 0)
    return MUTT_MBOX;
 
  FILE *fp = mutt_file_fopen(path, "r");
  if (!fp)
    return MUTT_UNKNOWN;
 
  int ch;
  while ((ch = fgetc(fp)) != EOF)
  {
    /* Some mailbox creation tools erroneously append a blank line to
     * a file before appending a mail message.  This allows neomutt to
     * detect type for and thus open those files. */
    if ((ch != '\n') && (ch != '\r'))
    {
      ungetc(ch, fp);
      break;
    }
  }
 
  enum MailboxType type = MUTT_UNKNOWN;
  char tmp[256] = { 0 };
  if (fgets(tmp, sizeof(tmp), fp))
  {
    if (mutt_str_startswith(tmp, "From "))
      type = MUTT_MBOX;
    else if (mutt_str_equal(tmp, MMDF_SEP))
      type = MUTT_MMDF;
  }
  mutt_file_fclose(&fp);
 
  const bool c_check_mbox_size = cs_subset_bool(NeoMutt->sub, "check_mbox_size");
  if (!c_check_mbox_size)
  {
    /* need to restore the times here, the file was not really accessed,
     * only the type was accessed.  This is important, because detection
     * of "new mail" depends on those times set correctly.  */
#ifdef HAVE_UTIMENSAT
    struct timespec ts[2] = { { 0 }, { 0 } };
    mutt_file_get_stat_timespec(&ts[0], &st, MUTT_STAT_ATIME);
    mutt_file_get_stat_timespec(&ts[1], &st, MUTT_STAT_MTIME);
    utimensat(AT_FDCWD, path, ts, 0);
#else
    struct utimbuf times = { 0 };
    times.actime = st->st_atime;
    times.modtime = st->st_mtime;
    utime(path, &times);
#endif
  }
 
  return type;
}

static int mbox_path_canon(struct Buffer *path)
{
  mutt_path_canon(path, NeoMutt->home_dir, false);
  return 0;
}

static int mbox_path_is_empty(struct Buffer *path)
{
  return mutt_file_check_empty(buf_string(path));
}

static int mmdf_msg_commit(struct Mailbox *m, struct Message *msg)
{
  if (fputs(MMDF_SEP, msg->fp) == EOF)
    return -1;
 
  if ((fflush(msg->fp) == EOF) || (fsync(fileno(msg->fp)) == -1))
  {
    mutt_perror(_("Can't write message"));
    return -1;
  }
 
  return 0;
}

static int mmdf_msg_padding_size(struct Mailbox *m)
{
  return 10;
}

static enum MxStatus mbox_mbox_check_stats(struct Mailbox *m, uint8_t flags)
{
  struct stat st = { 0 };
  if (stat(mailbox_path(m), &st) != 0)
    return MX_STATUS_ERROR;
 
  bool new_or_changed;
 
  const bool c_check_mbox_size = cs_subset_bool(NeoMutt->sub, "check_mbox_size");
  if (c_check_mbox_size)
  {
    new_or_changed = (st.st_size > m->size);
  }
  else
  {
    new_or_changed =
        (mutt_file_stat_compare(&st, MUTT_STAT_MTIME, &st, MUTT_STAT_ATIME) > 0) ||
        (m->newly_created &&
         (mutt_file_stat_compare(&st, MUTT_STAT_CTIME, &st, MUTT_STAT_MTIME) == 0) &&
         (mutt_file_stat_compare(&st, MUTT_STAT_CTIME, &st, MUTT_STAT_ATIME) == 0));
  }
 
  if (new_or_changed)
  {
    const bool c_mail_check_recent = cs_subset_bool(NeoMutt->sub, "mail_check_recent");
    if (!c_mail_check_recent ||
        (mutt_file_stat_timespec_compare(&st, MUTT_STAT_MTIME, &m->last_visited) > 0))
    {
      m->has_new = true;
    }
  }
  else if (c_check_mbox_size)
  {
    /* some other program has deleted mail from the folder */
    m->size = (off_t) st.st_size;
  }
 
  if (m->newly_created && ((st.st_ctime != st.st_mtime) || (st.st_ctime != st.st_atime)))
    m->newly_created = false;
 
  if (flags & MUTT_MAILBOX_CHECK_STATS)
  {
    struct MboxAccountData *adata = mbox_adata_get(m);
    if (adata && mutt_file_stat_timespec_compare(&st, MUTT_STAT_MTIME,
                                                 &adata->stats_last_checked) > 0)
    {
      bool old_peek = m->peekonly;
      mx_mbox_open(m, MUTT_QUIET | MUTT_NOSORT | MUTT_PEEK);
      mx_mbox_close(m);
      m->peekonly = old_peek;
      mutt_time_now(&adata->stats_last_checked);
    }
  }
 
  if (m->has_new || m->msg_new)
    return MX_STATUS_NEW_MAIL;
  return MX_STATUS_OK;
}

const struct MxOps MxMboxOps = {
  // clang-format off
  .type            = MUTT_MBOX,
  .name             = "mbox",
  .is_local         = true,
  .ac_owns_path     = mbox_ac_owns_path,
  .ac_add           = mbox_ac_add,
  .mbox_open        = mbox_mbox_open,
  .mbox_open_append = mbox_mbox_open_append,
  .mbox_check       = mbox_mbox_check,
  .mbox_check_stats = mbox_mbox_check_stats,
  .mbox_sync        = mbox_mbox_sync,
  .mbox_close       = mbox_mbox_close,
  .msg_open         = mbox_msg_open,
  .msg_open_new     = mbox_msg_open_new,
  .msg_commit       = mbox_msg_commit,
  .msg_close        = mbox_msg_close,
  .msg_padding_size = mbox_msg_padding_size,
  .msg_save_hcache  = NULL,
  .tags_edit        = NULL,
  .tags_commit      = NULL,
  .path_probe       = mbox_path_probe,
  .path_canon       = mbox_path_canon,
  .path_is_empty    = mbox_path_is_empty,
  // clang-format on
};

const struct MxOps MxMmdfOps = {
  // clang-format off
  .type            = MUTT_MMDF,
  .name             = "mmdf",
  .is_local         = true,
  .ac_owns_path     = mbox_ac_owns_path,
  .ac_add           = mbox_ac_add,
  .mbox_open        = mbox_mbox_open,
  .mbox_open_append = mbox_mbox_open_append,
  .mbox_check       = mbox_mbox_check,
  .mbox_check_stats = mbox_mbox_check_stats,
  .mbox_sync        = mbox_mbox_sync,
  .mbox_close       = mbox_mbox_close,
  .msg_open         = mbox_msg_open,
  .msg_open_new     = mbox_msg_open_new,
  .msg_commit       = mmdf_msg_commit,
  .msg_close        = mbox_msg_close,
  .msg_padding_size = mmdf_msg_padding_size,
  .msg_save_hcache  = NULL,
  .tags_edit        = NULL,
  .tags_commit      = NULL,
  .path_probe       = mbox_path_probe,
  .path_canon       = mbox_path_canon,
  .path_is_empty    = mbox_path_is_empty,
  // clang-format on
};