/**************************************************************************
*                     X J D X G E N
*                                                   Author: Jim Breen
*
*   This is the Unix version of JDXGEN, ported from MS-DOS
*      (Hitoshi Doi tidied up a little)
*
*  V1.2 by Roger Ang, April, '95
*  - ported to Amiga
*  - added memory saving routines
*  - re-indented
***************************************************************************/
#ifdef __AMIGA__
#include <exec/types.h>
#else
#include <sys/types.h>
#include <sys/stat.h>
#endif

#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
#include <string.h>

/* memory buffering stuff */
#include "stack_buf.c"

#define TRUE 1
#define FALSE 0

unsigned char *db;

#ifdef __AMIGA__
#include <libraries/dos.h>
#include <proto/dos.h>

struct FileLock *flock;
struct FileInfoBlock buf;
#else
struct stat *buf;
#endif

unsigned long dbyte, indlen;
unsigned long *jindex;
unsigned long indptr,lone;
unsigned char Dname[30] = {"edict"};
unsigned char JDXname[30] = {"edict.xjdx"};

/* words to exclude */
unsigned char wds[] ={"abbr col vul vulg then and conj int pron pref suf syn ant from adv adj hon hum pol jap japanese "};

unsigned char *exlist[100];		/* list of words to be excluded */
int excount,exlens[100];
int jiver = 14;	/*The last time the index structure changed was Version1.4*/

/* global for buffering dictionary file */
SB_bufferset *dict_buffers = NULL;
int percent_done = 0;  /* just do display progress of sort */

/*====== prototypes=================================================*/
int stringcomp(unsigned char *s1, unsigned char *s2);
void jqsort(long i, long j);
int alphaoreuc(unsigned char c);
int Kstrcmp(unsigned long lhs, unsigned long rhs);
void ExSet(void);
int alphaoreuc(unsigned char x);

unsigned long GetIndSize(unsigned long len);
void print_use(char *prog_name);
void jinsert(long insertp, long i);
void jmsort(long lhs, long rhs);

int stringcomp(unsigned char *s1, unsigned char *s2)
{
  int i;
  unsigned char c1,c2;
  
  for(i = 0; i < strlen(s1);i++) {
    c1 = s1[i];
    if (c1 < 0x60)    c1 = (c1|0x20);
    c2 = s2[i];
    if (c2 < 0x60)    c2 = (c2|0x20);
    if (c1 != c2) return(1);
  }

  return (0);
}

/*====function to Load Dictionary and load/create index table=======*/
int main(argc,argv)
int argc;
char **argv;
{
  FILE *fp,*fopen();
  unsigned long possav,schi,diclen;
  int i,inwd,cstrp,saving,isc,nodread;
  unsigned char c;
  unsigned char currstr[40];
  char *ap;
  int save_mem, orig_max;

  save_mem = 0;
  printf("\nXJDXGEN Index Table Generator for XJDIC. \n      Copyright J.W. Breen, 1993\n");
  printf("V1.2 by Roger Ang, April '95\n");

  if (argc > 3) {
    print_use(argv[0]);
    exit(0);
  }

  if (argc > 1) {
    i = 1;
    while (i < argc) {
      ap = argv[i];
      if (ap[0] == '-') {
        if (ap[1] == 'm') {
  	  save_mem = 1;
	}
        else {
          print_use(argv[0]);
          exit(0);
        }     
      }
      else {
	strcpy(Dname,ap);
	strcpy(JDXname,ap);
	strcat(JDXname,".xjdx");
	printf("Commandline request to use files %s and %s \n",Dname,JDXname);
      }
      i++;
    }
  }
  ExSet();
  inwd = FALSE;
  indptr = 1;
  lone = 1;
  
  /* get size of dictionary file in bytes, put in diclen */
#ifdef __AMIGA__
  flock = (struct FileLock *) Lock(Dname, SHARED_LOCK);
  if (flock == NULL)  {
    perror(NULL);
    printf("Cannot lock: %s \n",Dname);
    exit(0);
  }
  if (Examine((BPTR) flock, &buf) == NULL) {
    perror(NULL);
    printf("Cannot examine: %s \n",Dname);
    UnLock((BPTR) flock);
    exit(0);
  }
  UnLock((BPTR) flock);
  diclen = buf.fib_Size;
#else  /* not Amiga */
  buf = (void *)malloc(1000);
  if(stat(Dname, buf) != 0) {
    perror(NULL);
    printf("Cannot stat: %s \n",Dname);
    exit(0);
  }
  diclen = buf->st_size;
#endif
  
  printf("\nWARNING!!  This program may take a long time to run .....\n");
  
  if (!save_mem) {
    printf("\nLoading Dictionary file.  Please wait.....\n");
    fp=fopen(Dname,"rb");
    if (fp==NULL ) {
      printf("\nCannot open dictionary file\n");
      exit(0);
    }

    db = (unsigned char *)calloc(diclen+100, sizeof(unsigned char));

    if (db == NULL) {
      fprintf(stderr,"Couldn't allocate memory for dictionary.\n");
      fclose(fp);
      exit(0);
    } 
    nodread = diclen/1024;
    dbyte = fread((unsigned char *)db+1, 1024, nodread, fp);
    nodread = diclen % 1024;
    dbyte = fread((unsigned char *)(db+(diclen/1024)*1024)+1, nodread,1, fp);
    fclose(fp);
    diclen++;
    db[diclen] = 10;
    db[0] = 10;
  }
  else {
    diclen++; 
    db = NULL;
  }

  if (db == NULL) {
    /* setup buffering of variable */
    if (!SBuffer_var((void**) (&db))) {
      fprintf(stderr,"Could not memory buffer dictionary\n");
      exit(0);
    }
    else
      dict_buffers = var_SBuffered((void**) (&db));

    /* will only use 1 buffer for pre-scan */
    orig_max = SB_max_buffers;
    SB_max_buffers = 1;
  }

  dbyte = diclen;
  printf("Dictionary size: %ld bytes.\n", diclen);

  jindex = NULL;
  if (!save_mem)  {
    indlen = diclen * sizeof(unsigned long);
    jindex = (unsigned long *) calloc(diclen, sizeof(unsigned long));
  }
    
  if (jindex == NULL) {
    /* scan to get actual number of indexes needed */
    printf("Scanning.... \n");
    indlen = (GetIndSize(diclen) + 2) * sizeof(unsigned long);

    /* free all buffers, free up memory for index table */
    freeall_SBuffers();
    SB_max_buffers = orig_max;
    printf("Index table size: %ld bytes.\n",indlen);

    jindex = (unsigned long *) calloc((indlen/sizeof(unsigned long)),
                                      sizeof(unsigned long)); 
  
    if (jindex == NULL) {
      fprintf(stderr,"Couldn't allocate memory for index table.\n");
      exit(0);
    }
  }
  
  printf("Parsing.... \n");
  /* this is the dictionary parser. It places an entry in jindex for 
     every kana/kanji string and every alphabetic string it finds 
     which is >=3 characters and is not on the "exclude" list */
  indptr = 1;
  saving = FALSE;
  cstrp = 0;
  /* scan whole dictionary  */
  for (schi = 0; schi < dbyte; schi++) {

    if (!dict_buffers) {
      c = db[schi];
    }
    else {
      c = get_SBuffered_char(Dname, dict_buffers, schi, dbyte);
      if (c == 0)  {
        fprintf(stderr,"Memory buffer problem - index: %ld\n", schi);
        exit(0);
      }
    }

    if (inwd) {
      if ((alphaoreuc(c)) || (c == '-') || (c == '.') || 
          ((c >= '0')&&(c <= '9'))) {
	currstr[cstrp] = c;
	cstrp++;
      	  }
      else {
	currstr[cstrp] = '\0';
	inwd = FALSE;
	if ((strlen(currstr) <= 2) && (currstr[0] < 127))    saving = FALSE;
	if ((strlen(currstr) == 2) && (currstr[1] <= '9'))   saving = TRUE;
	if (saving && (currstr[0] > 127)) {
	  possav = jindex[indptr];
	  indptr++;
	  if (indptr > indlen/sizeof(long)) {
	    printf("Index table overflow. Dictionary too large?\n");
	    exit(0);
	  }
		  
	  for (i = 2; i < strlen(currstr); i+=2) {
	    if (currstr[i] > 0xa5) {
	      jindex[indptr] = possav+i;
	      indptr++;
	      if (indptr > indlen/sizeof(long)) {
		printf("Index table overflow. Dictionary too large?\n");
		exit(0);
	      }
	    }
	  }
	}
	if (saving && (currstr[0] < 127)) {
	  indptr++;
	  if (indptr > indlen/sizeof(long)) {
	    printf("Index table overflow. Dictionary too large?\n");
	    exit(0);
	  }
	  if (currstr[0] < 128) {
	    for (isc = 0; isc <= excount; isc++) {
	      if (( exlens[isc] == strlen(currstr)) &&
		  (stringcomp(currstr,exlist[isc]) == 0)) {
		indptr--;
		break;
	      }
	    }
	  }
	}
      }
    }
    else {
      if (alphaoreuc(c)) {
	inwd = TRUE;
	jindex[indptr] = schi;
	cstrp = 1;
	currstr[0] = c;
	currstr[1] = '\0';
	saving = TRUE;
      }
    }
  }
  indptr--;
  printf("Index entries: %ld  \nSorting (this is slow)......\n",indptr);
  if (dict_buffers) 
    jmsort(lone,indptr);
  else {
    jqsort(lone,indptr); 
    printf("\rSorted\nWriting index file ....\n");
    fp = fopen(JDXname,"wb");
    if (fp==NULL ) {
      printf("\nCannot open %s output file\n",JDXname);
      exit(0);
    }
    jindex[0] = diclen+jiver;
    fwrite((char*) jindex,sizeof(long),indptr+1,fp);
    fclose(fp);
  }

  freeall_SBuffers();
  while (SB_global_bufferset) {
    SB_global_bufferset = 
      SB_bufferset_remove(SB_global_bufferset, dict_buffers);
    SB_bufferset_delete(dict_buffers);
  }
  free(jindex);
  if (db)  free(db);

  return 0;
}

/*======function to sort jindex table====================*/
void jqsort(long lhs, long rhs)
{
  long i,last,midp;
  unsigned long temp;
  int percent;

  if (lhs >= rhs) return;

  if ((lhs+1) == rhs) {
    if (Kstrcmp(jindex[rhs],jindex[lhs]) < 0) {
      temp = jindex[rhs];
      jindex[rhs] = jindex[lhs];
      jindex[lhs] = temp;
    }
    return;
  }

  /* Swap ( lhs , (lhs+rhs)/2);*/
  midp = (lhs+rhs)/2;
  temp = jindex[lhs];
  jindex[lhs] = jindex[midp];
  jindex[midp] = temp;
  last = lhs;
  for (i = lhs+1;i <= rhs; i++) {
    if (Kstrcmp(jindex[i],jindex[lhs]) < 0) {
      /* Swap(++last,i);*/
      last++;
      temp = jindex[i];
      jindex[i] = jindex[last];
      jindex[last] = temp;
    }
  }

  /* Swap (lhs,last);*/
  temp = jindex[lhs];
  jindex[lhs] = jindex[last];
  jindex[last] = temp;

  jqsort(lhs,last-1);
  jqsort(last+1,rhs);

  percent = (rhs*100)/indptr;
  if (percent > percent_done) {
    printf("\r%d%%", percent);
    fflush(stdout);
    percent_done = percent;
  }
}
/*=====string comparison used by jqsort==========================*/
int Kstrcmp(unsigned long lhs, unsigned long rhs)
{
  int i,c1,c2;

  /* effectively does a strnicmp on two "strings" within the dictionary,
     except it will make katakana and hirgana match (EUC A4 & A5) */

  /* !!! NOTE: assumes strings of 20 */
  for (i = 0; i<20 ; i++) {
    if (!dict_buffers) {
      c1 = db[lhs+i];
      c2 = db[rhs+i];
    }
    else { 
      c1 = get_SBuffered_char(Dname, dict_buffers, lhs+i, dbyte);
      if (c1 == 0)  {
        fprintf(stderr, "Memory buffer problem - index: %ld\n", lhs+i);
        exit(0);
      }

      c2 = get_SBuffered_char(Dname, dict_buffers, rhs+i, dbyte);
      if (c2 == 0)  {
        fprintf(stderr, "Memory buffer problem - index: %ld\n", rhs+i);
        exit(0);
      }
    }

    if ((i % 2) == 0) {
      if (c1 == 0xA5)  {
	c1 = 0xA4;
      }
      if (c2 == 0xA5) {
	c2 = 0xA4;
      }
    }
    if ((c1 >= 'A') && (c1 <= 'Z'))   c1 |= 0x20;
    if ((c2 >= 'A') && (c2 <= 'Z'))   c2 |= 0x20;
    if (c1 != c2 ) break;
  }
  return (c1-c2);
}

/*=========function to set up the exclusion list for the index=========*/
void ExSet()
{
  unsigned char *ptr;
  
  int i;
  /*  WARNING - do not put more than 100 words into this list		*/
  ptr = (unsigned char *)strtok(wds," ");
  i = 0;
  while (ptr != NULL)  {
    exlist[i] = ptr;
    exlens[i] = strlen(exlist[i]);
    i++;
    ptr = (unsigned char *)strtok(NULL," ");
  }
  excount = i-1;
}
/*=======function to test a character for alpha or kana/kanji====*/
int alphaoreuc(unsigned char x)
{
  int c;
  
  c = x & 0xff;
  if(((c >= 65) && (c <= 90)) || ((c >= 97) && (c <= 122))) {
    return (TRUE);
  }
  if ((c >= '0') && (c <= '9')) {
    return(TRUE);
  }
  if ((c & 0x80) > 0)  {
    return(TRUE);
  }
  return (FALSE);
}


/* Basically, a stripped scan to count number of indexes needed */
unsigned long GetIndSize(len)
unsigned long len;
{
  unsigned long schi;
  int i,inwd,cstrp,saving,isc;
  unsigned char c;
  unsigned char currstr[40];
  
  inwd = FALSE;
  indptr = 1;
  saving = FALSE;
  cstrp = 0;
  
  /* scan whole dictionary  */
  for (schi =0; schi < len; schi++) {
    if (!dict_buffers)
      c = db[schi];
    else { 
      c = get_SBuffered_char(Dname, dict_buffers, schi, len);
      if (c == 0)  {
        fprintf(stderr,"Memory buffer problem - index: %ld\n", schi);
        exit(0);
      }
    }

    if (inwd) {
      if ((alphaoreuc(c))||(c == '-')||(c == '.')||
	  ((c >= '0')&&(c <= '9')))  {
	currstr[cstrp] = c;
	cstrp++;
      }
      else { /* not EUC, alpha, or number character */
	currstr[cstrp] = '\0';
	inwd = FALSE;
	if ((strlen(currstr) <= 2) && (currstr[0] < 127))saving = FALSE;
	if ((strlen(currstr) == 2) && (currstr[1] <= '9'))saving = TRUE;
	if (saving && (currstr[0] > 127)) {
	  indptr++;
	  for (i = 2; i < strlen(currstr); i+=2) 
	    if (currstr[i] > 0xa5) 
	      indptr++;
	}
	if (saving && (currstr[0] < 127)) {
	  indptr++;
	  /* check if word to be excluded */
	  if (currstr[0] < 128) 
	    for (isc = 0; isc <= excount; isc++) 
	      if ((exlens[isc] == strlen(currstr)) &&
		  (stringcomp(currstr,exlist[isc]) == 0)) {
		indptr--;
		break;
	      }
	}
      }    /* else not EUC, alpha, or number character */
    }
    else { /* not in a word */
      if (alphaoreuc(c)) {
        inwd = TRUE;
	cstrp = 1;
	currstr[0] = c;
	currstr[1] = '\0';
	saving = TRUE;
      }
    }
  }
  indptr--;
  return indptr;
}

void print_use(prog_name)
char *prog_name;
{
  printf("\nUsage: %s [-m] <dictionary>\n", prog_name);
  printf("    -m conserve memory.\n");
  printf("    <dictionary> a JDic dictionary file.\n");
}


/* merge sort index table - this is better than a quick sort when
   memory buffering of the files is used because indexes close together
   are sorted first.  Hopefully, this will reduce disk thrashing. - Roger */
void jmsort(long lhs, long rhs)
{
  FILE *fp, *fp2, *final;
  long  midp;
  unsigned long n, i1, i2;
  int r1, r2, at_top;

  r1 = r2 = at_top = 0;

  if (lhs >= rhs) return;
  /* got to be a least two elements to sort */

  midp = (lhs+rhs)/2;

  if ((jindex[midp]-jindex[lhs]) < 
      (SB_buffer_size*(dict_buffers->num_bufs-1))) {
    /* small enough to do in memory */
    jqsort(lhs,midp);
    jqsort(midp+1,rhs);
  }
  else {
    jmsort(lhs,midp);
    jmsort(midp+1,rhs);
  }

  /* dump each half */
  fp=fopen("xjdx.tmp1","wb");
  if (fp==NULL ) {
    printf("\nCannot open temporary file\n");
    exit(0);
  }
  fwrite((char*) (jindex+lhs),sizeof(long),(midp-lhs+1),fp);
  fclose(fp);

  fp=fopen("xjdx.tmp2","wb");
  if (fp==NULL ) {
    printf("\nCannot open dictionary file\n");
    exit(0);
  }
  fwrite((char*) (jindex+midp+1),sizeof(long),(rhs-midp),fp);
  fclose(fp);

  /* if at top level, free jindex.  Don't need it any more */
  at_top = ((lhs == 1) && (rhs == indptr));
  if (at_top)   free(jindex);

  /* merge lists */
  fp=fopen("xjdx.tmp1","rb");
  if (fp==NULL ) {
    printf("\nCannot open temporary file\n");
    exit(0);
  }

  fp2=fopen("xjdx.tmp2","rb");
  if (fp2==NULL ) {
    printf("\nCannot open temporary file\n");
    exit(0);
  }

  if (at_top) {
    final=fopen(JDXname,"wb");
    if (final==NULL ) {
      printf("\nCannot open index file\n");
      exit(0);
    }
  }

  n = lhs;
  if (at_top) {
    printf("\rFinal sort\nWriting index file ....\n");
    i1 = dbyte + jiver;
    fwrite((char*) &i1, sizeof(long), 1, final);
  }
  fread((char*) &i1, sizeof(long), 1, fp);
  fread((char*) &i2, sizeof(long), 1, fp2);
  while (!feof(fp) && !feof(fp2)) {
    if (Kstrcmp(i1, i2) <= 0) {
      if (at_top)
        fwrite((char*) &i1, sizeof(long), 1, final);
      else
        jindex[n] = i1;
      r1 = fread((char*) &i1, sizeof(long), 1, fp);
    }
    else {
      if (at_top)
        fwrite((char*) &i2, sizeof(long), 1, final);
      else
        jindex[n] = i2;
      r2 = fread((char*) &i2, sizeof(long), 1, fp2);
    }
    n++;
  }

  /* finish lists, note: only one of the two while loops should be true */
  while (!feof(fp)) {
    if (at_top)
      fwrite((char*) &i1, sizeof(long), 1, final);
    else
      jindex[n] = i1;
    r1 = fread((char*) &i1, sizeof(long), 1, fp);
    n++;
  }
  if (r1)      
    if (at_top)
      fwrite((char*) &i1, sizeof(long), 1, final);
    else
      jindex[n] = i1;

  while (!feof(fp2)) {
    if (at_top)
      fwrite((char*) &i2, sizeof(long), 1, final);
    else
      jindex[n] = i2;
    r2 = fread((char*) &i2, sizeof(long), 1, fp2);
    n++;
  }
  if (r2)      
    if (at_top)
      fwrite((char*) &i2, sizeof(long), 1, final);
    else
      jindex[n] = i2;

  fclose(fp);
  fclose(fp2);
  if (at_top)  fclose(final);

  /* remove tmp files */
  remove("xjdx.tmp1");
  remove("xjdx.tmp2");
}