hcache.c

Go to the documentation of this file.

 
#include "config.h"
#include <errno.h>
#include <limits.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#include "mutt/lib.h"
#include "config/lib.h"
#include "email/lib.h"
#include "core/lib.h"
#include "lib.h"
#include "compress/lib.h"
#include "store/lib.h"
#include "hcache/hcversion.h" // path needed by out-of-tree build
#include "module_data.h"
#include "muttlib.h"
#include "serialize.h"
 
#if !(defined(HAVE_GDBM) || defined(HAVE_LMDB) || defined(HAVE_ROCKSDB) || defined(HAVE_TDB))
#error "No hcache backend defined"
#endif

struct RealKey
{
  char key[1024]; 
  size_t keylen;  
};

static struct RealKey *realkey(struct HeaderCache *hc, const char *key,
                               size_t keylen, bool compress)
{
  static struct RealKey rk;
 
  int n = snprintf(rk.key, sizeof(rk.key), "%s/%.*s", hc->folder, (int) keylen, key);
  if (n < 0)
    n = 0;
  rk.keylen = MIN((size_t) n, sizeof(rk.key) - 1);
 
#ifdef USE_HCACHE_COMPRESSION
  if (compress && hc->compr_ops)
  {
    // Append the compression type, e.g. "-zstd"
    n = snprintf(rk.key + rk.keylen, sizeof(rk.key) - rk.keylen, "-%s",
                 hc->compr_ops->name);
    if (n > 0)
      rk.keylen += MIN((size_t) n, sizeof(rk.key) - 1 - rk.keylen);
  }
#endif
 
  return &rk;
}

static void hcache_free(struct HeaderCache **ptr)
{
  if (!ptr || !*ptr)
    return;
 
  struct HeaderCache *hc = *ptr;
  FREE(&hc->folder);
 
  FREE(ptr);
}

static struct HeaderCache *hcache_new(void)
{
  return MUTT_MEM_CALLOC(1, struct HeaderCache);
}

static size_t header_size(void)
{
  return sizeof(int) + sizeof(uint32_t);
}

static inline uint32_t email_pack_flags(const struct Email *e)
{
  if (!e)
    return 0;
 
  // clang-format off
  return e->security +
        (e->expired    << 16) +
        (e->flagged    << 17) +
        (e->mime       << 18) +
        (e->old        << 19) +
        (e->read       << 20) +
        (e->replied    << 21) +
        (e->superseded << 22) +
        (e->trash      << 23);
  // clang-format on
}

static inline void email_unpack_flags(struct Email *e, uint32_t packed)
{
  if (!e)
    return;
 
  // clang-format off
  e->security   = (packed & ((1 << 16) - 1)); // bits 0-15
  e->expired    = (packed & (1 << 16));
  e->flagged    = (packed & (1 << 17));
  e->mime       = (packed & (1 << 18));
  e->old        = (packed & (1 << 19));
  e->read       = (packed & (1 << 20));
  e->replied    = (packed & (1 << 21));
  e->superseded = (packed & (1 << 22));
  e->trash      = (packed & (1 << 23));
  // clang-format on
}

static inline uint32_t email_pack_timezone(const struct Email *e)
{
  if (!e)
    return 0;
 
  return e->zhours + (e->zminutes << 5) + (e->zoccident << 11);
}

static inline void email_unpack_timezone(struct Email *e, uint32_t packed)
{
  if (!e)
    return;
 
  // clang-format off
  e->zhours    =  (packed       & ((1 << 5) - 1)); // bits 0-4 (5)
  e->zminutes  = ((packed >> 5) & ((1 << 6) - 1)); // bits 5-10 (6)
  e->zoccident =  (packed       &  (1 << 11));     // bit  11 (1)
  // clang-format on
}

static void *dump_email(struct HeaderCache *hc, const struct Email *e, int *off, uint32_t uidvalidity)
{
  bool convert = !CharsetIsUtf8;
 
  *off = 0;
  unsigned char *d = MUTT_MEM_MALLOC(4096, unsigned char);
 
  d = serial_dump_uint32_t((uidvalidity != 0) ? uidvalidity : mutt_date_now(), d, off);
  d = serial_dump_int(hc->crc, d, off);
 
  ASSERT((size_t) *off == header_size());
 
  uint32_t packed = email_pack_flags(e);
  d = serial_dump_uint32_t(packed, d, off);
 
  packed = email_pack_timezone(e);
  d = serial_dump_uint32_t(packed, d, off);
 
  uint64_t big = e->date_sent;
  d = serial_dump_uint64_t(big, d, off);
  big = e->received;
  d = serial_dump_uint64_t(big, d, off);
 
  d = serial_dump_int(e->lines, d, off);
 
  d = serial_dump_envelope(e->env, d, off, convert);
  d = serial_dump_body(e->body, d, off, convert);
  d = serial_dump_tags(&e->tags, d, off);
 
  return d;
}

static struct Email *restore_email(const unsigned char *d, size_t dlen)
{
  int off = 0;
  struct Email *e = email_new();
  bool convert = !CharsetIsUtf8;
  uint32_t packed = 0;
  uint64_t big = 0;
  unsigned int num = 0;
 
  off += sizeof(uint32_t);     // skip validate
  off += sizeof(unsigned int); // skip crc
 
  if (!serial_restore_uint32_t(&packed, d, &off, dlen))
    goto fail;
  email_unpack_flags(e, packed);
 
  packed = 0;
  if (!serial_restore_uint32_t(&packed, d, &off, dlen))
    goto fail;
  email_unpack_timezone(e, packed);
 
  if (!serial_restore_uint64_t(&big, d, &off, dlen))
    goto fail;
  e->date_sent = big;
 
  big = 0;
  if (!serial_restore_uint64_t(&big, d, &off, dlen))
    goto fail;
  e->received = big;
 
  if (!serial_restore_int(&num, d, &off, dlen))
    goto fail;
  e->lines = num;
 
  e->env = mutt_env_new();
  if (!serial_restore_envelope(e->env, d, &off, dlen, convert))
    goto fail;
 
  e->body = mutt_body_new();
  if (!serial_restore_body(e->body, d, &off, dlen, convert))
    goto fail;
 
  if (!serial_restore_tags(&e->tags, d, &off, dlen))
    goto fail;
 
  return e;
 
fail:
  email_free(&e);
  return NULL;
}

static bool create_hcache_dir(const char *path)
{
  char *dir = mutt_str_dup(path);
  if (!dir)
    return false;
 
  char *p = strrchr(dir, '/');
  if (!p)
  {
    FREE(&dir);
    return true;
  }
 
  *p = '\0';
 
  int rc = mutt_file_mkdir(dir, S_IRWXU | S_IRWXG | S_IRWXO);
  if (rc != 0)
    mutt_error(_("Can't create %s: %s"), dir, strerror(errno));
 
  FREE(&dir);
  return (rc == 0);
}

static void hcache_per_folder(struct HeaderCache *hc, struct Buffer *hcpath,
                              const char *path, hcache_namer_t namer)
{
  struct stat st = { 0 };
 
  int plen = mutt_str_len(path);
  int rc = stat(path, &st);
  bool slash = (path[plen - 1] == '/');
 
  if (((rc == 0) && !S_ISDIR(st.st_mode)) || ((rc == -1) && !slash))
  {
    /* An existing file or a non-existing path not ending with a slash */
    mutt_encode_path(hcpath, path);
    create_hcache_dir(buf_string(hcpath));
    return;
  }
 
  /* We have a directory - no matter whether it exists, or not */
  struct Buffer *hcfile = buf_pool_get();
  if (namer)
  {
    namer(hc->store_ops, hc->folder, hcfile);
    mutt_encode_path(hcfile, buf_string(hcfile));
    buf_concat_path(hcpath, path, buf_string(hcfile));
  }
  else
  {
    unsigned char m[16] = { 0 }; /* binary md5sum */
    struct Buffer *name = buf_pool_get();
 
#ifdef USE_HCACHE_COMPRESSION
    const char *cm = hc->compr_ops ? hc->compr_ops->name : "";
    buf_printf(name, "%s|%s%s", hc->store_ops->name, hc->folder, cm);
#else
    buf_printf(name, "%s|%s", hc->store_ops->name, hc->folder);
#endif
    mutt_md5(buf_string(name), m);
    buf_reset(name);
    mutt_md5_toascii(m, name->data);
    mutt_encode_path(name, buf_string(name));
    buf_printf(hcpath, "%s%s%s", path, slash ? "" : "/", buf_string(name));
    buf_pool_release(&name);
  }
 
  create_hcache_dir(buf_string(hcpath));
  buf_pool_release(&hcfile);
}

static char *get_foldername(const char *folder)
{
  /* if the folder is local, canonify the path to ensure equivalent paths share
   * the hcache */
  char *p = MUTT_MEM_MALLOC(PATH_MAX + 1, char);
  if (!realpath(folder, p))
    mutt_str_replace(&p, folder);
 
  return p;
}

static void free_raw(struct HeaderCache *hc, void **data)
{
  hc->store_ops->free(hc->store_handle, data);
}

static unsigned int generate_hcachever(void)
{
  union
  {
    unsigned char charval[16]; 
    unsigned int intval;       
  } digest = { 0 };
  struct Md5Ctx md5ctx = { 0 };
 
  mutt_md5_init_ctx(&md5ctx);
 
  /* Seed with the compiled-in header structure hash */
  unsigned int ver = HCACHEVER;
  mutt_md5_process_bytes(&ver, sizeof(ver), &md5ctx);
 
  struct EmailModuleData *mod_data = neomutt_get_module_data(NeoMutt, MODULE_ID_EMAIL);
  ASSERT(mod_data);
 
  /* Mix in user's spam list */
  struct Replace *sp = NULL;
  STAILQ_FOREACH(sp, &mod_data->spam, entries)
  {
    mutt_md5_process(sp->regex->pattern, &md5ctx);
    mutt_md5_process(sp->templ, &md5ctx);
  }
 
  /* Mix in user's nospam list */
  struct RegexNode *np = NULL;
  STAILQ_FOREACH(np, &mod_data->no_spam, entries)
  {
    mutt_md5_process(np->regex->pattern, &md5ctx);
  }
 
  /* Get a hash and take its bytes as an (unsigned int) hash version */
  mutt_md5_finish_ctx(&md5ctx, digest.charval);
 
  return digest.intval;
}

struct HeaderCache *hcache_open(const char *path, const char *folder,
                                hcache_namer_t namer, bool create)
{
  if (!path || (path[0] == '\0'))
    return NULL;
 
  struct HcacheModuleData *mod_data = neomutt_get_module_data(NeoMutt, MODULE_ID_HCACHE);
  if (mod_data->hcache_ver == 0x0)
    mod_data->hcache_ver = generate_hcachever();
 
  struct HeaderCache *hc = hcache_new();
 
  hc->folder = get_foldername(folder);
  hc->crc = mod_data->hcache_ver;
 
  const char *const c_header_cache_backend = cs_subset_string(NeoMutt->sub, "header_cache_backend");
  hc->store_ops = store_get_backend_ops(c_header_cache_backend);
  if (!hc->store_ops)
  {
    hcache_free(&hc);
    return NULL;
  }
 
#ifdef USE_HCACHE_COMPRESSION
  const char *const c_header_cache_compress_method = cs_subset_string(NeoMutt->sub, "header_cache_compress_method");
  if (c_header_cache_compress_method)
  {
    hc->compr_ops = compress_get_ops(c_header_cache_compress_method);
 
    const short c_header_cache_compress_level = cs_subset_number(NeoMutt->sub, "header_cache_compress_level");
    hc->compr_handle = hc->compr_ops->open(c_header_cache_compress_level);
    if (!hc->compr_handle)
    {
      hcache_free(&hc);
      return NULL;
    }
 
    /* remember the buffer of database backend */
    mutt_debug(LL_DEBUG3, "Header cache will use %s compression\n",
               hc->compr_ops->name);
  }
#endif
 
  struct Buffer *hcpath = buf_pool_get();
  hcache_per_folder(hc, hcpath, path, namer);
 
  hc->store_handle = hc->store_ops->open(buf_string(hcpath), create);
  if (!hc->store_handle)
  {
    /* remove a possibly incompatible version */
    if (unlink(buf_string(hcpath)) == 0)
    {
      hc->store_handle = hc->store_ops->open(buf_string(hcpath), create);
    }
  }
 
  if (!hc->store_handle)
  {
    if (hc->compr_ops)
    {
      hc->compr_ops->close(&hc->compr_handle);
    }
    hcache_free(&hc);
  }
 
  buf_pool_release(&hcpath);
  return hc;
}

void hcache_close(struct HeaderCache **ptr)
{
  if (!ptr || !*ptr)
    return;
 
  struct HeaderCache *hc = *ptr;
 
#ifdef USE_HCACHE_COMPRESSION
  if (hc->compr_ops)
    hc->compr_ops->close(&hc->compr_handle);
#endif
 
  hc->store_ops->close(&hc->store_handle);
 
  hcache_free(ptr);
}

struct HCacheEntry hcache_fetch_email(struct HeaderCache *hc, const char *key,
                                      size_t keylen, uint32_t uidvalidity)
{
  struct HCacheEntry hce = { 0 };
  if (!hc)
    return hce;
 
  size_t dlen = 0;
  struct RealKey *rk = realkey(hc, key, keylen, true);
  void *data = hc->store_ops->fetch(hc->store_handle, rk->key, rk->keylen, &dlen);
  void *to_free = data;
  if (!data)
  {
    goto end;
  }
 
  /* restore uidvalidity and crc */
  size_t hlen = header_size();
  if (hlen > dlen)
  {
    goto end;
  }
  int off = 0;
  serial_restore_uint32_t(&hce.uidvalidity, data, &off, dlen);
  serial_restore_int(&hce.crc, data, &off, dlen);
  ASSERT((size_t) off == hlen);
  if ((hce.crc != hc->crc) || ((uidvalidity != 0) && (uidvalidity != hce.uidvalidity)))
  {
    goto end;
  }
 
#ifdef USE_HCACHE_COMPRESSION
  if (hc->compr_ops)
  {
    size_t ulen = 0;
    void *dblob = hc->compr_ops->decompress(hc->compr_handle, (char *) data + hlen,
                                            dlen - hlen, &ulen);
    if (!dblob)
    {
      goto end;
    }
    data = (char *) dblob - hlen; /* restore skips uidvalidity and crc */
    dlen = hlen + ulen;
  }
#endif
 
  hce.email = restore_email(data, dlen);
 
end:
  free_raw(hc, &to_free);
  return hce;
}

bool hcache_fetch_raw_obj_full(struct HeaderCache *hc, const char *key,
                               size_t keylen, void *dst, size_t dstlen)
{
  if (!hc)
    return false;
 
  bool rc = true;
  size_t srclen = 0;
 
  struct RealKey *rk = realkey(hc, key, keylen, false);
  void *src = hc->store_ops->fetch(hc->store_handle, rk->key, rk->keylen, &srclen);
 
  if (src && (srclen == dstlen))
  {
    memcpy(dst, src, dstlen);
  }
  else
  {
    rc = false;
  }
  free_raw(hc, &src);
  return rc;
}

char *hcache_fetch_raw_str(struct HeaderCache *hc, const char *key, size_t keylen)
{
  if (!hc)
    return NULL;
 
  char *res = NULL;
  size_t dlen = 0;
 
  struct RealKey *rk = realkey(hc, key, keylen, false);
  void *data = hc->store_ops->fetch(hc->store_handle, rk->key, rk->keylen, &dlen);
  if (data)
  {
    res = mutt_strn_dup(data, dlen);
    free_raw(hc, &data);
  }
  return res;
}

int hcache_store_email(struct HeaderCache *hc, const char *key, size_t keylen,
                       struct Email *e, uint32_t uidvalidity)
{
  if (!hc)
    return -1;
 
  int dlen = 0;
  char *data = dump_email(hc, e, &dlen, uidvalidity);
 
#ifdef USE_HCACHE_COMPRESSION
  if (hc->compr_ops)
  {
    /* We don't compress uidvalidity and the crc, so we can check them before
     * decompressing on fetch().  */
    size_t hlen = header_size();
 
    /* data / dlen gets ptr to compressed data here */
    size_t clen = dlen;
    void *cdata = hc->compr_ops->compress(hc->compr_handle, data + hlen, dlen - hlen, &clen);
    if (!cdata)
    {
      FREE(&data);
      return -1;
    }
 
    char *whole = MUTT_MEM_MALLOC(hlen + clen, char);
    memcpy(whole, data, hlen);
    memcpy(whole + hlen, cdata, clen);
 
    FREE(&data);
 
    data = whole;
    dlen = hlen + clen;
  }
#endif
 
  struct RealKey *rk = realkey(hc, key, keylen, true);
  int rc = hc->store_ops->store(hc->store_handle, rk->key, rk->keylen, data, dlen);
 
  FREE(&data);
 
  return rc;
}

int hcache_store_raw(struct HeaderCache *hc, const char *key, size_t keylen,
                     void *data, size_t dlen)
{
  if (!hc)
    return -1;
 
  struct RealKey *rk = realkey(hc, key, keylen, false);
  int rc = hc->store_ops->store(hc->store_handle, rk->key, rk->keylen, data, dlen);
 
  return rc;
}

int hcache_delete_email(struct HeaderCache *hc, const char *key, size_t keylen)
{
  if (!hc)
    return -1;
 
  struct RealKey *rk = realkey(hc, key, keylen, true);
 
  return hc->store_ops->delete_record(hc->store_handle, rk->key, rk->keylen);
}

int hcache_delete_raw(struct HeaderCache *hc, const char *key, size_t keylen)
{
  if (!hc)
    return -1;
 
  struct RealKey *rk = realkey(hc, key, keylen, false);
 
  return hc->store_ops->delete_record(hc->store_handle, rk->key, rk->keylen);
}