/*$Id: ezmlm-archive.c,v 1.13 1999/11/28 20:13:32 lindberg Exp $*/
/*$Name: ezmlm-idx-040 $*/

#include "alloc.h"
#include "error.h"
#include "stralloc.h"
#include "str.h"
#include "sig.h"
#include "getconf.h"
#include "strerr.h"
#include "getln.h"
#include "substdio.h"
#include "readwrite.h"
#include "fmt.h"
#include "sgetopt.h"
#include "idxthread.h"
#include "makehash.h"
#include "idx.h"
#include "errtxt.h"

#define FATAL "ezmlm-archive: fatal: "
#define WARNING "ezmlm-archive: warning: inconsistent index: "

substdio ssin;
char inbuf[1024];
substdio ssout;
char outbuf[1024];
substdio ssnum;
char numbuf[16];

stralloc line = {0};
stralloc num = {0};
stralloc fn = {0};
stralloc fnn = {0};

char strnum[FMT_ULONG];
int flagerror = 0;
int flagsync = 1;	/* sync() by default, not for -c or -f or -t */
char *dir;

struct ca {
  char *s;		/* start */
  unsigned int l;	/* length */
} ca;

void die_usage() {
  strerr_die1x(100,
    "ezmlm-archive: usage: "
	"ezmlm-archive [-cCFsSTvV] [-f min_msg] [-t max_msg] dir");
}

void die_nomem() { strerr_die2x(111,FATAL,ERR_NOMEM); }

void close_proper(ss,s,sn)
/* flush,sync,close,move sn->s) */
substdio *ss;
char *s, *sn;
{
   if (substdio_flush(ss) == -1)
     strerr_die6sys(111,FATAL,ERR_FLUSH,dir,"/",s,": ");
  if (flagsync)
    if (fsync(ss->fd) == -1)
       strerr_die6sys(111,FATAL,ERR_SYNC,dir,"/",s,": ");
  if (close(ss->fd) == -1)
     strerr_die6sys(111,FATAL,ERR_CLOSE,dir,"/",s,": ");
  if (rename(sn,s) == -1)
     strerr_die6sys(111,FATAL,ERR_MOVE,dir,"/",sn,": ");
}

void write_threads(msgtable,subtable,authtable,datetable,from,to)
/* Add the current threading data to the thread database without dups */
/* Writes the subject index first, then processes the individual files */
msgentry *msgtable; subentry *subtable; authentry *authtable;
dateentry *datetable;
unsigned long from,to;
{
  msgentry *pmsgt;
  subentry *psubt,*psubtm, *psubtlast;
  subentry *presubt = (subentry *)0;
  authentry *pautht;
  dateentry *pdatet;
  char *cp,*cp1;
  unsigned long msg;
  unsigned long ulmsginthread;
  unsigned long subnum;
  unsigned long authnum;
  unsigned long msgnum;
  unsigned int pos,l;
  unsigned int startdate,nextdate;
  unsigned int startmsg,nextmsg;
  int fd = -1;
  int fdn = -1;
  int match;
  int ffound;
  int lineno;
  int res;

  psubtm = subtable;		/* now for new threads */
  pdatet = datetable;
  nextmsg = 0L;
  nextdate = pdatet->date;
  while (psubtm->sub) {		/* these are in msgnum order */
    if (!presubt)		/* for rewind */
      if (psubtm->lastmsg >= nextmsg)
	presubt = psubtm;	/* this thread extends beyond current month */
    if (psubtm->firstmsg >= nextmsg) {	/* done with this month */
      if (fdn != -1) close_proper(&ssout,fn.s,fnn.s);
      psubtlast = psubtm;		/* last thread done */
      if (presubt)			/* need to rewind? */
	psubtm = presubt;		/* do it */
      psubt = psubtm;			/* tmp pointer to reset done flag */
      presubt = (subentry *)0;		/* reset rewind pointer */
      pdatet++;				/* next month */
      startdate = nextdate;		/* startdate */
      nextdate = pdatet->date;		/* end date */
      startmsg = nextmsg;		/* first message in month */
      nextmsg = pdatet->msg;		/* first message in next month */
      if (!stralloc_copys(&fn,"archive/threads/")) die_nomem();
      if (!stralloc_catb(&fn,strnum,fmt_uint(strnum,startdate))) die_nomem();
      if (!stralloc_copy(&fnn,&fn)) die_nomem();
      if (!stralloc_0(&fn)) die_nomem();
      if (!stralloc_cats(&fnn,"n")) die_nomem();
      if (!stralloc_0(&fnn)) die_nomem();
      if ((fdn = open_trunc(fnn.s)) == -1)
	strerr_die6sys(111,FATAL,ERR_CREATE,dir,"/",fnn.s,": ");
      substdio_fdbuf(&ssout,write,fdn,outbuf,sizeof(outbuf));
      if ((fd = open_read(fn.s)) == -1) {
      if (errno != error_noent)
             strerr_die6sys(111,FATAL,ERR_OPEN,dir,"/",fn.s,": ");
      } else {
	substdio_fdbuf(&ssin,read,fd,inbuf,sizeof(inbuf));
      for (;;) {
      if (getln(&ssin,&line,&match,'\n') == -1)
             strerr_die6sys(111,FATAL,ERR_READ,dir,"/",fn.s,": ");
      if (!match) break;
      pos = scan_ulong(line.s,&msgnum);
      pos++;			/* skip ':' */
      if (msgnum >= from)
	continue;		/* ignore entries from threading range */
      if (line.len < pos + HASHLEN) {
	flagerror = -1;		/* and bad ones */
	continue;
      }
      psubt = subtable;
      cp = line.s + pos;
      ffound = 0;		/* search among already known subjects */
      for (;;) {
	res = str_diffn(psubt->sub,cp,HASHLEN);
	if (res < 0) {
	  if (psubt->higher)
	    psubt = psubt->higher;
	 else
	   break;
	} else if (res > 0) {
	  if (psubt->lower)
	    psubt = psubt->lower;
	  else
	    break;
	} else {
	  ffound = 1;
	  break;
	}
      }
      if (!ffound) {
	if (substdio_put(&ssout,line.s,line.len) == -1)
	  strerr_die6sys(111,FATAL,ERR_WRITE,dir,"/",fnn.s,": ");
      } else {			/* new # of msg in thread */
	cp += HASHLEN;		/* HASHLEN [#] Subject always \n at end */
	if (*(cp++) == ' ' && *(cp++) == '[') {
	  cp += scan_ulong(cp,&ulmsginthread);
	  if (*cp == ']') {
	    psubt->msginthread += (unsigned char) (ulmsginthread & 0xff);
	  }
	} else
	  flagerror = -5;
      }
    }
    close(fd);
  }
  continue;
  }

    if (psubtm->firstmsg < nextmsg && psubtm->lastmsg >= startmsg) {
    if (!stralloc_copyb(&line,strnum,fmt_ulong(strnum,psubtm->lastmsg)))
		die_nomem();
    if (!stralloc_cats(&line,":")) die_nomem();
    if (!stralloc_catb(&line,psubtm->sub,HASHLEN)) die_nomem();
    if (!stralloc_cats(&line," [")) die_nomem();
    if (!stralloc_catb(&line,strnum,
	fmt_ulong(strnum,(unsigned long) psubtm->msginthread)))
		die_nomem();
    if (!stralloc_cats(&line,"]")) die_nomem();
    if (!stralloc_catb(&line,psubtm->sub + HASHLEN,psubtm->sublen - HASHLEN))
			 die_nomem();	/* has \n */
    if (substdio_put(&ssout,line.s,line.len) == -1)
	strerr_die6sys(111,FATAL,ERR_WRITE,dir,"/",fnn.s,": ");
    }
  psubtm++;
  }
  if (fdn != -1)
    close_proper(&ssout,fn.s,fnn.s);

  psubt = subtable;
  while (psubt->sub) {		/* now the threads */
    if (!stralloc_copys(&fn,"archive/subjects/")) die_nomem();
    if (!stralloc_catb(&fn,psubt->sub,2)) die_nomem();
    if (!stralloc_0(&fn)) die_nomem();
    if (mkdir(fn.s,0755) == -1)
    if (errno != error_exist)
      strerr_die6sys(111,FATAL,ERR_CREATE,dir,"/",fn.s,": ");
    fn.s[fn.len - 1] = '/';
    if (!stralloc_catb(&fn,psubt->sub+2,HASHLEN-2)) die_nomem();
    if (!stralloc_copy(&fnn,&fn)) die_nomem();
    if (!stralloc_cats(&fnn,"n")) die_nomem();
    if (!stralloc_0(&fn)) die_nomem();
    if (!stralloc_0(&fnn)) die_nomem();
    if ((fdn = open_trunc(fnn.s)) == -1)
      strerr_die4sys(111,FATAL,ERR_CREATE,fnn.s,": ");
    substdio_fdbuf(&ssout,write,fdn,outbuf,sizeof(outbuf));
    if ((fd = open_read(fn.s)) == -1) {
      if (errno != error_noent)
	  strerr_die4sys(111,FATAL,ERR_OPEN,fn.s,": ");
      if (substdio_puts(&ssout,psubt->sub) == -1)	/* write subject */
	     strerr_die6sys(111,FATAL,ERR_WRITE,dir,"/",fnn.s,": ");
    } else {					/* copy data */
	substdio_fdbuf(&ssin,read,fd,inbuf,sizeof(inbuf));
	lineno = 0;
	for (;;) {
	  if (getln(&ssin,&line,&match,'\n') == -1)
             strerr_die6sys(111,FATAL,ERR_READ,dir,"/",fn.s,": ");
          if (!match) break;
	  if (!lineno) {			/* write subject */
	    if (line.len < HASHLEN + 1 || line.s[HASHLEN] != ' ')
		flagerror = -3;
	    if (substdio_put(&ssout,line.s,line.len) == -1)
	       strerr_die6sys(111,FATAL,ERR_WRITE,dir,"/",fnn.s,": ");
	    lineno = 1;
	    continue;
	  }
	  (void) scan_ulong(line.s,&msgnum);
	  if (msgnum >= from) break;
	  if (substdio_put(&ssout,line.s,line.len) == -1)
	     strerr_die6sys(111,FATAL,ERR_WRITE,dir,"/",fnn.s,": ");
	}
	(void) close(fd);	/* close old index */
      }

    subnum = (unsigned long) (psubt - subtable + 1);	/* idx of this subj */
    pmsgt = msgtable + psubt->firstmsg - from;	/* first message entry */
    for (msg = psubt->firstmsg; msg <= psubt->lastmsg; msg++) {
      if (pmsgt->subnum == subnum) {
        if (!stralloc_copyb(&line,strnum,fmt_ulong(strnum,msg))) die_nomem();
        if (!stralloc_cats(&line,":")) die_nomem();
	if (!stralloc_catb(&line,strnum,fmt_uint(strnum,pmsgt->date)))
		die_nomem();
	if (!stralloc_cats(&line,":")) die_nomem();
        if (pmsgt->authnum) {
	  pautht = authtable + pmsgt->authnum - 1;
	  cp = pautht->auth;
	  cp1 = cp + str_chr(cp,' ');
	  if (cp + HASHLEN != cp1)
	    strerr_die1x(100,ERR_BAD_INDEX);
	  if (!stralloc_cats(&line,cp))
		die_nomem();				/* hash */
	} else
          if (!stralloc_cats(&line,"\n")) die_nomem();
	if (substdio_put(&ssout,line.s,line.len) == -1)
	  strerr_die6sys(111,FATAL,ERR_WRITE,dir,"/",fnn.s,": ");
      }
      pmsgt++;
    }
    close_proper(&ssout,fn.s,fnn.s);
    psubt++;
  }

					/* (no master author index) */
  pautht = authtable;
  while (pautht->auth) {		/* now the authors */
    if (!stralloc_copys(&fn,"archive/authors/")) die_nomem();
    if (!stralloc_catb(&fn,pautht->auth,2)) die_nomem();
    if (!stralloc_0(&fn)) die_nomem();
    if (mkdir(fn.s,0755) == -1)
    if (errno != error_exist)
      strerr_die6sys(111,FATAL,ERR_CREATE,dir,"/",fn.s,": ");
    fn.s[fn.len - 1] = '/';
    if (!stralloc_catb(&fn,pautht->auth+2,HASHLEN-2)) die_nomem();
    if (!stralloc_copy(&fnn,&fn)) die_nomem();
    if (!stralloc_cats(&fnn,"n")) die_nomem();
    if (!stralloc_0(&fn)) die_nomem();
    if (!stralloc_0(&fnn)) die_nomem();
    if ((fdn = open_trunc(fnn.s)) == -1)
      strerr_die4sys(111,FATAL,ERR_CREATE,fnn.s,": ");
    substdio_fdbuf(&ssout,write,fdn,outbuf,sizeof(outbuf));
      if ((fd = open_read(fn.s)) == -1) {
	if (errno != error_noent)
	  strerr_die4sys(111,FATAL,ERR_OPEN,fn.s,": ");
        else {			/* didn't exist before: write author */
          if (substdio_put(&ssout,pautht->auth,pautht->authlen) == -1)
	     strerr_die6sys(111,FATAL,ERR_WRITE,dir,"/",fnn.s,": ");
	}
      } else {					/* copy data */
	substdio_fdbuf(&ssin,read,fd,inbuf,sizeof(inbuf));
	lineno = 0;
	for (;;) {
	  if (getln(&ssin,&line,&match,'\n') == -1)
             strerr_die6sys(111,FATAL,ERR_READ,dir,"/",fn.s,": ");
          if (!match) break;
	  if (!lineno) {			/* write author */
	    if (line.len < HASHLEN + 1 || line.s[HASHLEN] != ' ')
		flagerror = - 4;
	    if (substdio_put(&ssout,line.s,line.len) == -1)
	       strerr_die6sys(111,FATAL,ERR_WRITE,dir,"/",fnn.s,": ");
	    lineno = 1;
	    continue;
	  }
	  (void) scan_ulong(line.s,&msgnum);
	  if (msgnum >= from) break;
	  if (substdio_put(&ssout,line.s,line.len) == -1)
	     strerr_die6sys(111,FATAL,ERR_WRITE,dir,"/",fnn.s,": ");
	}
	(void) close(fd);			/* close old index */
      }

    authnum = (unsigned long) (pautht - authtable + 1);	/* idx of this auth */
    pmsgt = msgtable + pautht->firstmsg - from;	/* first message entry */
    for (msg = pautht->firstmsg; msg <= to; msg++) {
      if (pmsgt->authnum == authnum) {
        if (!stralloc_copyb(&line,strnum,fmt_ulong(strnum,msg))) die_nomem();
        if (!stralloc_cats(&line,":")) die_nomem();
	if (!stralloc_catb(&line,strnum,fmt_uint(strnum,pmsgt->date)))
		die_nomem();
	if (!stralloc_cats(&line,":")) die_nomem();
        if (pmsgt->subnum) {
	  psubt = subtable + pmsgt->subnum - 1;
          if (!stralloc_catb(&line,psubt->sub,psubt->sublen))
		die_nomem();
	}
	if (substdio_put(&ssout,line.s,line.len) == -1)
	  strerr_die6sys(111,FATAL,ERR_WRITE,dir,"/",fnn.s,": ");
      }
      pmsgt++;
    }
    close_proper(&ssout,fn.s,fnn.s);
    pautht++;
  }
}

int main(argc,argv)
int argc;
char **argv;
{
  unsigned long archnum = 0L;
  unsigned long to = 0L;
  unsigned long max;
  int fd;
  int fdlock;
  int flagcreate = 0;
  int flagsyncall = 0;
  int opt;
  msgentry *msgtable;
  subentry *subtable;
  authentry *authtable;
  dateentry *datetable;

  (void) umask(022);
  sig_pipeignore();

  while ((opt = getopt(argc,argv,"cCf:FsSt:TvV")) != opteof)
    switch (opt) {
      case 'c':	flagcreate = 1;
		flagsync = 0;
		break;			/* start at beginning of archive */
      case 'C': flagcreate = 0;
		break;	/* Do only archnum+1 => num */
      case 'f': if (optarg) {
		  (void) scan_ulong(optarg,&archnum);
		  archnum = (archnum / 100) * 100;
	        }
		flagsync = 0;
		break;
      case 'F': archnum = 0; break;
      case 's': flagsyncall = 1; break;
      case 'S': flagsyncall = 0; break;
      case 't': if (optarg) {
		  (void) scan_ulong(optarg,&to);
		}
		flagsync = 0;
		break;
      case 'T': to = 0; break;
      case 'v':
      case 'V': strerr_die2x(0,"ezmlm-archive version: ",EZIDX_VERSION);
      default:
        die_usage();
    }

  if (flagsyncall) flagsync = 1;	/* overrides */
  dir = argv[optind++];
  if (!dir) die_usage();
  if (chdir(dir) == -1)
    strerr_die4sys(111,FATAL,ERR_SWITCH,dir,": ");

  if (mkdir("archive/threads",0755) == -1)
    if (errno != error_exist)
      strerr_die4sys(111,FATAL,ERR_CREATE,dir,"/archive/threads: ");
  if (mkdir("archive/subjects",0755) == -1)
    if (errno != error_exist)
      strerr_die4sys(111,FATAL,ERR_CREATE,dir,"/archive/subjects: ");
  if (mkdir("archive/authors",0755) == -1)
    if (errno != error_exist)
      strerr_die4sys(111,FATAL,ERR_CREATE,dir,"/archive/authors: ");

	/* Lock list to assure that no ezmlm-send is working on it */
	/* and that the "num" message is final */
  fdlock = open_append("lock");
  if (fdlock == -1)
    strerr_die2sys(111,FATAL,ERR_OPEN_LOCK);
  if (lock_ex(fdlock) == -1) {
    (void) close(fdlock);
    strerr_die2sys(111,FATAL,ERR_OBTAIN_LOCK);
  }
					/* get num */
  if (!getconf_line(&num,"num",0,FATAL,dir))
    strerr_die1x(100,ERR_EMPTY_LIST);
  (void) close(fdlock);

  if (!stralloc_0(&num)) die_nomem();	/* parse num */
  (void) scan_ulong(num.s,&max);
  if (!to || to > max) to = max;

  fdlock = open_append("archive/lock");	/* lock index */
  if (fdlock == -1)
    strerr_die4sys(111,FATAL,ERR_OPEN,dir,"/archive/lock: ");
  if (lock_ex(fdlock) == -1) {
    (void) close(fdlock);
    strerr_die4sys(111,FATAL,ERR_OBTAIN,dir,"/archive/lock: ");
  }
  if (!flagcreate && !archnum) {	/* adjust archnum (from) / to */
    if (getconf_line(&num,"archnum",0,FATAL,dir)) {
      if (!stralloc_0(&num)) die_nomem();
      (void) scan_ulong(num.s,&archnum);
      archnum++;
    }
  }

  if (archnum > to)
    _exit(0);				/* nothing to do */

					/* do the subject threading */
  idx_mkthreads(&msgtable,&subtable,&authtable,&datetable,
	archnum,to,max,0,FATAL);
					/* update the index */
  write_threads(msgtable,subtable,authtable,datetable,archnum,to);
					/* update archnum */
  if ((fd = open_trunc("archnumn")) == -1)
    strerr_die4sys(111,FATAL,ERR_CREATE,dir,"/archnumn: ");
  substdio_fdbuf(&ssnum,write,fd,numbuf,sizeof(numbuf));
  if (substdio_put(&ssnum,strnum,fmt_ulong(strnum,to)) == -1)
     strerr_die6sys(111,FATAL,ERR_WRITE,dir,"/",fnn.s,": ");
  if (substdio_puts(&ssnum,"\n") == -1)
     strerr_die6sys(111,FATAL,ERR_WRITE,dir,"/",fnn.s,": ");
  close_proper(&ssnum,"archnum","archnumn");
  switch (flagerror) {
    case 0:
       _exit(0);				/* go bye-bye */
    case -1:
       strerr_die2x(99,WARNING,"threads entry with illegal format");
    case -2:
       strerr_die2x(99,WARNING,"thread in index, but threadfile missing");
    case -3:
       strerr_die2x(99,WARNING,"a subject file lacks subject");
    case -4:
       strerr_die2x(99,WARNING,"an author file lacks author/hash");
    case -5:
       strerr_die2x(99,WARNING,"threads entry lacks message count");
    default:
       strerr_die2x(99,WARNING,"something happened that isn't quite right");
  }
}