/*(( "Header" */
/*
 * $Id: httpproxy.c,v 0.12 1996/02/12 19:23:36 mshopf Exp mshopf $
 *
 * (c) 1995 Matthias Hopf
 *
 * A small little Http Proxy.
 * It can serv as a ProxyProxy, too (i.e. it can perform only caching and will
 * get its data from another proxy).
 * That way it can be used for other protocol types than html, too.
 *
 * Run it standalone at high priority. It won't need much computing time as
 * it does no busy wait at all.
 * If you really want to re-get an already cached page, just reload it immedeately.
 * (no other request inbetween and no more than ReloadTime seconds delay).
 * If you browse offline, you'll get a note that the cache is invalid. Reload the
 * page if you want to queue the page and get the old cache.
 */

/*
 * $Log: httpproxy.c,v $
 * Revision 0.12  1996/02/12  19:23:36  mshopf
 * assert() to logfile and debugfile and stderr.
 * MaxRequests now variable.
 * ReadCacheList()/SaveCacheList().
 * lots of bug fixes and cosmetic changes.
 *
 * Revision 0.11  1996/01/15  22:27:58  mshopf
 * fixed AmiTCP4.0 broken time() / stat() times (fix by Fionn Behrens).
 * another couple of bug fixes.
 * added dynamic cache table allocation.
 * error replies still getting better.
 * removed host realname lookup.
 *
 * Revision 0.10  1996/01/09  17:20:35  mshopf
 * Three major and some other minor bug fixes.
 * Added UrlBuffer for POST method and non-proxyproxy support.
 * More and more comformant error messages.
 * basic POST support (only proxying, no caching, no queuing).
 * Sending data even when request is not completed yet.
 * Recreating URL even from nonconformant requests.
 * Long requests supported now.
 * Some cosmetic changes.
 *
 * Revision 0.9  1995/12/06  19:53:55  mshopf
 * added fixes for unix machines and AmiTCP4.0 compilation.
 * some bug fixes.
 * more html conform now.
 * lots of printf type fixes.
 *
 * Revision 0.8  1995/12/03  14:20:23  mshopf
 * Made auto requests and proxy messages more http conform.
 *
 * Revision 0.7  1995/11/19  17:57:50  mshopf
 * added revbump compatible version string.
 * fixed ftp offline proxyproxy std port bug.
 *
 * Revision 0.6  1995/11/04  11:26:13  mshopf
 * better shutdown (requeueing of current transmissions). small bug fixes.
 *
 * Revision 0.5  1995/11/02  18:26:13  mshopf
 * queueing system implemented.
 *
 * Revision 0.4  1995/10/21  21:28:37  mshopf
 * small bug fix.
 *
 * Revision 0.3  1995/10/17  19:23:09  mshopf
 * cleaned up messy logging system.
 * new option 'log'.
 *
 * Revision 0.2  1995/10/13  18:09:17  mshopf
 * everything works so far, exceptions are noted in the header.
 * :-)
 *
 * Revision 0.1  1995/10/13  10:44:27  mshopf
 * no caching at all right now, but it works fine as a proxyproxy.
 *
 */


/*)) */
/*(( "Includes" */

#include "httpproxy_rev.h"
#include <stdio.h>
#include <stdlib.h>
#include <stddef.h>
#include <errno.h>
#include <string.h>
#include <unistd.h>
#include <time.h>
#include <ctype.h>

#include <syslog.h>
#include <dirent.h>
#include <sys/stat.h>
#include <sys/syslog.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <netdb.h>
#include <netinet/in.h>
#include <arpa/inet.h>

#ifdef _AMIGA
#  include <bsdsocket.h>
#  define ioctl IoctlSocket
#  define ioctl_t long
#  define strcasecmp stricmp
#  ifdef sys_errlist
#    define strerror(x) (sys_errlist [x])     /* Won't necessary for AmiTCP4.0 */
#  endif
#  define CURRENTDIR ""
typedef long len_t;

#  ifdef FIXTIME
/* Fix broken AmiTCP4.0 (or SasC+AmiTCP4.0) time() / stat() */
#    include <proto/dos.h>

time_t time (time_t *TimePtr)
{
    struct DateStamp ds;
    time_t t;
    DateStamp (&ds);

	/* Sekunden + Minuten*60 +
	 * (Tage + offset zu Unix time()[2922 Tage]) * 60 * 60 * 24  */
    t = ds.ds_Tick / 50+ds.ds_Minute*60+(ds.ds_Days+2922)*86400;
    if (TimePtr)
	*TimePtr = t;
    return (t);
}
#  endif

#else
#  define ioctl_t int
#  define CloseSocket close
#  define CURRENTDIR "."
typedef int len_t;
#endif

#ifdef DEBUG
#  define debug(x)   printf x
#else
#  define debug(x)   ((void) 0)
#endif

/* own assert() writing assertation to logfile */
#ifndef NDEBUG
#  define assert(x)  ASSERT ((int)(x), #x, __FILE__, __LINE__)
void ASSERT (int x, const char *text, const char *file, int line)
{
    extern FILE *LogStream;
    if (x)
	return;
    if (! text)
	text = "unknown";
    fprintf (stderr, "assertion (%s) failed in '%s' on line %d\n", text, file, line);
    debug   ((       "assertion (%s) failed in '%s' on line %d\n", text, file, line));
    if (LogStream)
    {
	fprintf (LogStream, "assertion (%s) failed in '%s' on line %d\n", text, file, line);
	fclose  (LogStream);
	LogStream = NULL;
    }
    exit (20);
    /*NOTREACHED*/
}
#else
#  define assert(x)  ((void) 0)
#endif

/*)) */
/*(( "Types / Variables / Constants" */

/* Types and constants */

#define NAME_CACHETABLE   ".cachetable"
#define DEFAULT_PROXYPORT 8080
#define DEFAULT_HTTPPORT  80
#define DEFAULT_DELTIME  (48*60*60)    /* default: delete cached files on startup */
#define DEFAULT_EXPIRETIME (24*60*60)  /* default: delete cache, when the page is requested and too old */
#define DEFAULT_RELOADTIME 10          /* default: expire cache on reload inbetween */
#define DEFAULT_MAX_REQUESTS 8         /* Maximum number of pending requests */

#define MIN_REQUESTS     4        /* Minimum number of free requests for interactive actions */
#define MAX_CACHES       32       /* maximum number of cache slots per cachearray */
#define MAX_FILENAME     20       /* maximum size of filename (>= 20) */
#define MAX_URLSAVE      128      /* maximum size of saved url requests */
#define MAX_URLBUFFER    512      /* no Url of any request may be larger than this value (<=MAX_REQBUFFER) */
#define MAX_REQBUFFER    1024
#define MAX_DATABUFFER   2048
#define SHIFT_DATABUFFER 512      /* when x bytes are in the data buffer, shift it after send() */
#define SHIFT_REQBUFFER  512      /* dto. for the request buffer */

#define REQ_REQSOCKET    0x01     /* a request socket is open */
#define REQ_CONNSOCKET   0x02     /* a connection socket is open / to be connected */
#define REQ_DONE         0x04     /* the connection socket is already closed */
#define REQ_REQDONE      0x08     /* the URL is completely read (cache may be sent) */
#define REQ_HTTP1X0      0x10     /* the URL was sent with HTTP/1.0 */
#define REQ_URLDONE      0x40     /* the URL request (the first) line is already there */

#define CACHE_VALID      0x01     /* Cache is filled and ready to serve */
#define CACHE_QUEUED     0x02     /* Cache is already queued for regetting */
#define CACHE_DELETETMP  0x04     /* There's a temporaray cache file to be removed */

#ifndef FALSE
#define FALSE            (0)
#define TRUE             (1)
#endif
				  /* Needed Memory */
#define NEED_MEM         ((MaxRequests * sizeof (request_t) + sizeof (cachearray_t)) / 1024)

#define difftime(x,y)    (((u_long)(x)) - ((u_long)(y)))
#define isvalidhttp(x)   ((unsigned char)(x) > ' ' && (x) != '%' && (x) != '\\' && (unsigned char)(x) < 128)

/*
 * Some typical Flag combinations in typical order:
 * None:                           Empty slot
 * REQ_REQSOCKET:                  Awaiting URL
 * REQ_REQSOCKET | REQ_CONNSOCKET: Still geting URL, but 1. line is already there
 *                                   *or*  URL done
 *                                 Awaiting connection to remote host
 *                                   *or*  already transfering data
 * REQ_REQSOCKET | REQ_DONE:       The connection socket is already closed, but data is
 *                                   still to be delivered, or data is sent from the cache
 * REQ_CONNSOCKET:                 Request was terminated, still geting data to fill up
 *                                   the cache (TIMEOUT needed!!!)
 */

/* REQ_REQSOCKET: alone will never occour on ProxyProxy==TRUE... */


typedef struct {
		   int  Flags;
		   char File [MAX_FILENAME];        /* "" if free slot / with begining '_' (data) */
		   char Url  [MAX_URLSAVE];         /* "" if free slot or just getting data */
	       } cache_t;

typedef struct cachearray {
			      struct cachearray *Next;
			      cache_t Caches [MAX_CACHES];
			  } cachearray_t;

typedef struct {
		   long  ReqSocket, ConnSocket;
		   int   Flags;
		   char  ReqBuffer  [MAX_REQBUFFER];
		   char  UrlBuffer  [MAX_URLBUFFER];/* This one contains the URL line for sending the request */
		   char  DataBuffer [MAX_DATABUFFER];
		   int   ReqSent, ReqRecv;
		   int   UrlSent, UrlRecv;
		   int   DataSent, DataRecv;
		   cache_t *Cache;                  /* NULL if not cacheable */
		   FILE  *Stream;                   /* != NULL on open data transfer to/from cache */
	       } request_t;


/* Global variables */

char      *VVersion= VERSTAG;
char      *Version = VERSTRING;
FILE      *LogStream = NULL;
char      *PrgName;
long      ServerSocket;
int       StdHttpPort;
int       ServerPort    = DEFAULT_PROXYPORT;
int       ProxyProxy    = FALSE;                    /* TRUE, when all requests should be
						     * forwarded to another proxy. */
struct sockaddr_in ProxyProxyIn;

cache_t   *LastCache    = NULL;
time_t    ThisRequestTime = 0;
time_t    StartUpTime;
int       MaxRequests   = DEFAULT_MAX_REQUESTS;
int       RequestsFree  = -1;
int       CachesFree    = MAX_CACHES;
long      CacheNr       = 0;
u_long    DelCacheTime  = DEFAULT_DELTIME;
u_long    ExpireCacheTime = DEFAULT_EXPIRETIME;
u_long    ReloadCacheTime = DEFAULT_RELOADTIME;
int       OffLine = 0;                              /* 0: get cache files again, when they are too old */
						    /* 1: keep cache files and queue them */
int       GetQueued = 0;                            /* 1: get queued data from remote hosts or proxyproxy */
int       CacheUnreadRequests = FALSE;              /* 1: keep cache data on data connection close with data in url send buffer */
request_t *Requests;
cachearray_t *Caches;


void DeleteConnect (request_t *Req, int ok);
void SaveCacheUrl (cache_t *c);

/*)) */

/*(( "Init ()" */

/* Init all global variables, open server port */

void Init (char *ProxyProxyHost, int ProxyProxyPort, char *LogName)
{
    struct servent  *ServEnt;
    struct sockaddr_in SockIn;
    struct hostent  *HostEnt;
    ioctl_t on = 1;

	/* Standard setups */

    if (! (LogStream = fopen (LogName, "a+")) )
    {
	fprintf (stderr, "cannot open logfile '%s': %s\n", LogName, strerror (errno));
	exit (20);
    }

    if (! (Requests = calloc (MaxRequests, sizeof (request_t))) )
    {
	fprintf (stderr, "not enough memory (need %d Kbyte)\n", NEED_MEM);
	exit (20);
    }
    RequestsFree = MaxRequests;

    if (! (Caches = calloc (1, sizeof (cachearray_t))) )
    {
	fprintf (stderr, "not enough memory (need %d Kbyte)\n", NEED_MEM);
	exit (20);
    }
    assert (Caches->Next == NULL);

    ThisRequestTime = StartUpTime = time (NULL);

	/* Get standard HTTP port (80 at the time of development...but you never know) */

    if (! (ServEnt = getservbyname ("http", "tcp")) )
    {
	StdHttpPort = DEFAULT_HTTPPORT;
	syslog (LOG_WARNING, "%s: unknown protocol 'http', using default port %d", PrgName, DEFAULT_HTTPPORT);
    }
    else
	StdHttpPort = ServEnt->s_port;

	/* Create Serversocket */

    if ( (ServerSocket = socket (PF_INET, SOCK_STREAM, 0)) < 0)
    {
	fprintf (stderr, "socket() for serverport failed: %s\n", strerror (errno));
	exit (20);
    }

    memset ((char *) &SockIn, 0, sizeof (struct sockaddr_in));
    SockIn.sin_family      = AF_INET;
    SockIn.sin_addr.s_addr = INADDR_ANY;
    SockIn.sin_port        = htons (ServerPort);
    if (bind (ServerSocket, (struct sockaddr *) &SockIn, sizeof (struct sockaddr_in)) < 0)
    {
	fprintf (stderr, "bind() failed for port %d: %s\n", ServerPort, strerror (errno));
	exit (20);
    }

#ifdef FIOASYNC
    if (ioctl (ServerSocket, FIOASYNC, (caddr_t) &on) < 0)
    {
	fprintf (stderr, "ioctl() failed for FIOASYNC: %s\n", strerror (errno));
	exit (20);
    }
#endif
#ifdef FIONBIO
    if (ioctl (ServerSocket, FIONBIO, (caddr_t) &on) < 0)
    {
	fprintf (stderr, "ioctl() failed for FIONBIO: %s\n", strerror (errno));
	exit (20);
    }
#endif

    if (listen (ServerSocket, 4) < 0)
    {
	fprintf (stderr, "listen() failed: %s\n", strerror (errno));
	exit (20);
    }

	/* Get ProxyProxy address */

    if (ProxyProxyHost)
    {
	ProxyProxy = TRUE;

	if (! OffLine)
	{
	    memset ((char *) &ProxyProxyIn, 0, sizeof (struct sockaddr_in));
	    ProxyProxyIn.sin_family     = AF_INET;
	    ProxyProxyIn.sin_port       = htons (ProxyProxyPort);

	    if ( (ProxyProxyIn.sin_addr.s_addr = inet_addr (ProxyProxyHost)) +0 == -1)                          /* !!! */
	    {
		if (! (HostEnt = gethostbyname (ProxyProxyHost)) )
		{
		    fprintf (stderr, "can't get proxyproxy host '%s'\n", ProxyProxyHost);
		    exit (20);
		}
		else
		    memmove (&ProxyProxyIn.sin_addr, HostEnt->h_addr, sizeof (struct in_addr));
	    }
	    fprintf (LogStream, "%s\nstarting with proxyproxy host '%s', port %d\n",
		     Version, ProxyProxyHost, ProxyProxyPort);
	    debug (("%s\nstarting with proxyproxy host '%s', port %d\n",
		    Version, ProxyProxyHost, ProxyProxyPort));
	    return;
	}
    }
    else if (! OffLine)
    {
	fprintf (LogStream, "%s\nstarting in normal mode\n", Version);
	debug (("%s\nstarting in normal mode\n", Version));
	return;
    }

    fprintf (LogStream, "%s\nstarting in offline mode\n", Version);
    debug  (("%s\nstarting in offline mode\n", Version));
}


/*)) */
/*(( "InitCacheSlot()/GetFreeCacheSlot()/ErrToReq()" */

/* Initialize cache slot */

void InitCacheSlot (cache_t *c, cache_t *Template, char Init, char Separator)
{
    if (! Template)
    {
	c->Flags  = 0;
	c->Url[0] = '\0';
	sprintf (c->File, "%c%08lx%c%08lx", Init, StartUpTime, Separator, CacheNr++);
	debug (("new cache allocated: '%s'\n", c->File));
    }
    else
    {
	c->Flags  = Template->Flags & ~CACHE_VALID;
	strcpy  (c->Url, Template->Url);
	sprintf (c->File, "%c%8.8s%c%8.8s", Init, & Template->File [1], Separator, & Template->File [10]);
	debug (("cache %s from template %s\n", c->File, Template->File));
    }
}


/* Scan for a free cache slot */

cache_t *GetFreeCacheSlot (void)
{
    cache_t *c;
    cachearray_t *a = Caches;
    int i;

    debug (("new cache requested.\n"));
    if (! CachesFree)
    {
	while (a->Next)
	    a = a->Next;

	if (! (a->Next = calloc (1, sizeof (cachearray_t))) )
	{
	    syslog (LOG_NOTICE, "%s: no more memory for cache arrays", PrgName);
	    return (NULL);
	}
	assert (a->Next->Next == NULL);
	debug (("allocated %d new cache slots at 0x%x\n", MAX_CACHES, a->Next));
	a = a->Next;
	CachesFree += MAX_CACHES - 1;
	c = & a->Caches [0];
	InitCacheSlot (c, NULL, '_', '.');
	return (& a->Caches [0]);
    }

    while (a)
    {
	for (c = a->Caches, i=0; i < MAX_CACHES; c++, i++)
	    if (c->Url[0] == '\0' && c->File[0] == '\0')
	    {
		CachesFree--;
		InitCacheSlot (c, NULL, '_', '.');
		return (c);
	    }
	a = a->Next;
    }
    assert (0);      /*NOTREACHED*/
}


/* Type an error to a data buffer of a pending request and set everything up.
 * The error is printed into the Logstream, too. When Short == NULL, a information
 * message is sent, and nothing is printed into the Logstream. */

void   ErrToReq (request_t *Req, int Number, char *Short, char *Descr)
{
    static char *Author  = "<A HREF=\"http://wwwcip.informatik.uni-erlangen.de/user/mshopf\">Matthias Hopf</A>";
    Req->DataSent = 0;

    if (Req->Flags & REQ_HTTP1X0)
    {
	if (Short)
	{
	    if (errno > 0)
	    {
		fprintf (LogStream, "#%02d: Error: %s: %s\n", Req-Requests, Short, strerror (errno));
		debug (("#%02d: Error: %s: %s\n", Req-Requests, Short, strerror (errno)));
	    }
	    else
	    {
		fprintf (LogStream, "#%02d: Error: %s\n", Req-Requests, Short);
		debug (("#%02d: Error: %s\n", Req-Requests, Short));
	    }

	    sprintf (Req->DataBuffer, "HTTP/1.0 %03d %s\r\n"
		     "Server: %s\r\n"
		     "Content-Type: text/html\r\n\r\n"
		     "<HTML><HEAD><TITLE>Proxy Error</TITLE></HEAD>\n"
		     "<BODY><H1>Proxy Error: %s</H1><P>\n"
		     "%s%s%s"
		     "%s<P>\n"
		     "<HR><ADDRESS>%s by %s</ADDRESS>\n"
		     "</BODY></HTML>\n",
		     Number, Short, VERSHTTP, Short,
		     (errno > 0 ? "<H3>Cause: " : ""),
		     (errno > 0 ? strerror (errno) : ""),
		     (errno > 0 ? "</H3>\n" : ""),
		     Descr, Version, Author);
	}
	else
	    sprintf (Req->DataBuffer, "HTTP/1.0 %03d Proxy Message\r\n"
		     "Server: %s\r\n"
		     "Content-Type: text/html\r\n\r\n"
		     "<HTML><HEAD><TITLE>Proxy Message</TITLE></HEAD>\n"
		     "<BODY><H3>Proxy Message:</H3><P>\n"
		     "%s<P>\n"
		     "<HR><ADDRESS>%s by %s</ADDRESS>\n"
		     "</BODY></HTML>\n",
		     Number, VERSHTTP, Descr, Version, Author);
    }
    else
    {
	if (Short)
	{
	    fprintf (LogStream, "#%02d: Error for non-Http1.0 request: %s\n", Req-Requests, Short);
	    debug (("#%02d: Error for non-Http1.0 request : %s\n", Req-Requests, Short));

	    sprintf (Req->DataBuffer, "<HTML><HEAD><TITLE>Proxy Error</TITLE></HEAD>\n"
		     "<BODY><H1>Proxy Error: %s</H1><P>\n"
		     "%s%s%s"
		     "%s<P>\n"
		     "<HR><ADDRESS>%s by %s</ADDRESS>\n"
		     "</BODY></HTML>\n",
		     Short,
		     (errno > 0 ? "<H3>Cause: " : ""),
		     (errno > 0 ? strerror (errno) : ""),
		     (errno > 0 ? "</H3>\n" : ""),
		     Descr, Version, Author);
	}
	else
	    sprintf (Req->DataBuffer, "<HTML><HEAD><TITLE>Proxy Message</TITLE></HEAD>\n"
		     "<BODY><H3>Proxy Message:</H3><P>\n"
		     "%s<P>\n"
		     "<HR><ADDRESS>%s by %s</ADDRESS>\n"
		     "</BODY></HTML>\n",
		     Descr, Version, Author);
    }

    Req->DataRecv = strlen (Req->DataBuffer);
    Req->Flags |= REQ_DONE;
    if (Req->Flags & REQ_CONNSOCKET)
	DeleteConnect (Req, FALSE);
}


/*)) */
/*(( "RemCacheEntry()/CheckCacheTime()" */

/* Remove a cache entry and its according files */

void RemCacheEntry (cache_t *c)
{
    cache_t tmp;

    debug (("deleting cache file '%s', additional url: %s\n", c->File, c->Url[0] ? "yes" : "no"));
    if (c->File[0] && c->File[0] != '.')                     /* It's no special message file */
    {
	if (remove (c->File))
	{
	    fprintf (LogStream, "cannot delete invalid cache file '%s': %s\n", c->File, strerror (errno));
	    debug (("cannot delete invalid cache file '%s': %s\n", c->File, strerror (errno)));
	}
	if (c->Url[0])
	{
	    c->File[0] = '@';
	    if (remove (c->File))
	    {
		fprintf (LogStream, "cannot delete invalid cache file '%s': %s\n", c->File, strerror (errno));
		debug (("cannot delete invalid cache file '%s': %s\n", c->File, strerror (errno)));
	    }
	}
	if (c->Flags & CACHE_DELETETMP)
	{
	    InitCacheSlot (&tmp, c, '@', '@');
	    debug (("removing temporary url file '%s'\n", tmp.File));
	    if (remove (tmp.File))
	    {
		fprintf (LogStream, "error while removing temporary url file '%s': %s\n", tmp.File, strerror (errno));
		debug (("error while removing temporary url file '%s': %s\n", tmp.File, strerror (errno)));
	    }
	}
    }

    c->File[0] = c->Url[0] = '\0';
    c->Flags = 0;
    CachesFree++;
}


/* Check the modification time and date of a cache entry; set Time to 0 to force expire. */
/* Returns -1, when the cache entry is expired or to be reloaded or not existing, the entry is removed.
 * Returns -2, when the cache entry is expired and queued. */

int CheckCacheTime (cache_t *c, u_long Time)
{
    time_t FileT = -1;
    struct stat s;

    if (stat (c->File, &s) >= 0)
    {
	FileT = s.st_mtime;
	debug (("local 0x%lx, access 0x%lx, diff1 %ld <?> %s\n", ThisRequestTime, FileT,
		difftime (ThisRequestTime, FileT), difftime (ThisRequestTime, FileT) > Time ? "expired" : "valid"));
	if ((! Time) || difftime (ThisRequestTime, FileT) > Time)
	{
	    if (OffLine)
	    {
		if ((c->Flags & CACHE_VALID) && ! (c->Flags & CACHE_QUEUED))
		{                                   /* When CACHE_VALID is not set, the routine was called from buildcache() or similar */
		    cache_t tmp;
		    c->Flags |= CACHE_QUEUED;
		    InitCacheSlot (&tmp, c, '@', '@'); /* Queue Url for cache entry and return -2 */
		    SaveCacheUrl (&tmp);
		}
		return (-2);
	    }
	    else if (c->Flags & CACHE_QUEUED)       /* Is the entry queued and not yet sent? */
		return (OffLine ? -2 : -1);
	    else
		FileT = -1;
	}
    }

    if (FileT == -1)
    {
	if (c->Flags & CACHE_QUEUED)
	{
	    debug (("cache entry '%s' not yet there (queued) - This should not happen...\n", c->File));
	    fprintf (stderr, "%s: Warning! Data consistency failure, line %d\n", PrgName, __LINE__);
	    return (-2);
	}
	else
	{
	    debug (("cache entry '%s' expired / to be reloaded, removing\n", c->File));
	    RemCacheEntry (c);
	    return (-1);
	}
    }
    return (c->Flags & CACHE_QUEUED ? -2 : 0);
}

/*)) */
/*(( "ReadCacheUrl()/SaveCacheUrl()" */

/* Read the Url from a specific url file */

void ReadCacheUrl (cache_t *c, char *Name)
{
    FILE *f;

    c->Url[0] = ' ';              /* RemCacheEntry shall remove the url file, too, in case it is called */
    switch (CheckCacheTime (c, DelCacheTime)) {
    case 0:
    case -3:
	break;

    case -1:
	return;
    default:
	RemCacheEntry (c);
	return;
    }

    if (! (f = fopen (Name, "r")) )
    {
	fprintf (LogStream, "cannot open url file '%s', removing cache entry: %s\n", Name, strerror (errno));
	debug (("cannot open url file '%s', removing cache entry: %s\n", Name, strerror (errno)));
	RemCacheEntry (c);
	return;
    }
    if (fgets (c->Url, MAX_URLSAVE, f))
    {
	switch (c->Url [strlen (c->Url) - 1]) {
	case '\n':
	case '\r':
	    break;

	default:                      /* Cache entry is valid -> return */
	    fclose (f);
	    c->Flags = CACHE_VALID;
	    debug (("valid cache entry: url '%s', data '%s'\n", c->Url, c->File));
	    return;
	}
    }
    fclose (f);
    fprintf (LogStream, "corrupt cache entry for url file '%s', removing\n", Name);
    debug (("corrupt cache entry for url file '%s', removing\n", Name));
    RemCacheEntry (c);
}


/* Save the cache's Url to its url file */

void SaveCacheUrl (cache_t *c)
{
    FILE *UrlStream;
    cache_t tmp;

    c->File[0] = '@';
    debug (("writing url file '%s'\n", c->File));
    if ( (UrlStream = fopen (c->File, "w")) )
    {
	fprintf (UrlStream, "%s", c->Url);  /* no error checking - well... */
	fclose (UrlStream);
	c->File[0] = '_';
	if (! (c->Flags & CACHE_QUEUED))
	    c->Flags |= CACHE_VALID;
	    /* fprintf (LogStream, "URL '%s' is now cached\n",
	     * c->Url);*/
	if (c->Flags & CACHE_DELETETMP)
	{
	    InitCacheSlot (&tmp, c, '@', '@');
	    debug (("removing temporary url file '%s'\n", tmp.File));
	    if (remove (tmp.File))
	    {
		fprintf (LogStream, "error while removing temporary url file '%s': %s\n", tmp.File, strerror (errno));
		debug (("error while removing temporary url file '%s': %s\n", tmp.File, strerror (errno)));
	    }
	    c->Flags &= ~CACHE_DELETETMP;
	}
    }
    else
    {
	fprintf (LogStream, "cannot write cache url file '%s': %s\n",
		 c->File, strerror (errno));
	debug (("cannot write cache url file '%s': %s\n",
		c->File, strerror (errno)));
	c->File[0] = '_';
	if (c->Flags & CACHE_QUEUED)
	    c->File[0] = '\0';
	RemCacheEntry (c);
    }
}


/*)) */
/*(( "ReadCacheList()/SaveCacheList()" */

/* Read the cache list file when it's there */
/* Format: Fields (max size):
 * Filename (MAX_FILENAME) " " Flags (1) Url (MAX_URLSAVE)
 * Terminated with \n. */
/* Flags:   v:Valid   q:Queued   r:Requeued (expired cache entry exists) */
/* <- -1: Failure   0: ok */

int ReadCacheList (void)
{
    cache_t *c;
    char Buffer [MAX_URLSAVE+MAX_FILENAME+3];
    FILE *Fd;
    char *p, *End;

    if (! (Fd = fopen (NAME_CACHETABLE, "r")))
	return -1;

    debug (("Cachetable valid.\n"));
    while (fgets (Buffer, MAX_URLSAVE+MAX_FILENAME+3, Fd))
    {
	End = Buffer + strlen (Buffer) - 1;
	*End = '\0';                                  /* Remove terminating '\n' */
	p = strchr (Buffer, ' ');
	if (p == NULL || p >= & Buffer [MAX_FILENAME])
	{
	    fprintf (LogStream, "Invalid line in cachetable, skipping: '%s'\n", Buffer);
	    debug (("Invalid line in cachetable, skipping: '%s'\n", Buffer));
	    continue;
	}
	p[0] = '\0';
	if (p[1] != 'v' && p[1] != 'q' && p[1] != 'r')
	{
	    fprintf (LogStream, "Unknown cache typ in cachetable, skipping: '%s'\n", Buffer);
	    debug (("Unknown cache typ in cachetable, skipping: '%s'\n", Buffer));
	    continue;
	}
	if (p[2] == '\0')
	{
	    fprintf (LogStream, "Invalid line in cachetable, skipping: '%s'\n", Buffer);
	    debug (("Invalid line in cachetable, skipping: '%s'\n", Buffer));
	    continue;
	}
	assert (&p[2] < End);
	assert (strlen (&p[2]) < MAX_URLSAVE);

	if (! (c = GetFreeCacheSlot ()))
	    break;

	strcpy (c->File, Buffer);
	strcpy (c->Url,  &p[2]);
	if (p[1] == 'v')
	    c->Flags = CACHE_VALID;
	else if (p[1] == 'q')
	    c->Flags = CACHE_QUEUED;
	else
	    c->Flags = CACHE_VALID | CACHE_QUEUED | CACHE_DELETETMP;
	debug (("read '%c' line from cachetable, file '%s', url '%s'\n", p[1], c->File, c->Url));
    }

    fclose (Fd);
    remove (NAME_CACHETABLE);
    return 0;
}


/* Save all cache entries in the cachetable file */

void SaveCacheList (void)
{
    cachearray_t *a;
    cache_t      *c;
    int          i;
    FILE         *Fd;

    if (! (Fd = fopen (NAME_CACHETABLE, "w")))
	return;

    for (a = Caches; a; a = a->Next)
	for (i = 0, c = a->Caches; i < MAX_CACHES; i++, c++)
	    if (c->File[0] && c->Url[0])
	    {
		if ((c->Flags & CACHE_VALID) && (c->Flags & CACHE_QUEUED))
		{
		    if (c->File[0] == '_')
			fprintf (Fd, "%s r%s\n", c->File, c->Url);
		    else
		    {
			fprintf (LogStream, "Warning! Requeued cache consistency error, file '%s', url '%s'\n", c->File, c->Url);
			fprintf (stderr,    "Warning! Requeued cache consistency error, file '%s', url '%s'\n", c->File, c->Url);
			debug             (("Warning! Requeued cache consistency error, file '%s', url '%s'\n", c->File, c->Url));
		    }
		}
		else if (c->Flags & CACHE_VALID)
		{
		    if (c->File[0] == '_')
			fprintf (Fd, "%s v%s\n", c->File, c->Url);
		    else
		    {
			fprintf (LogStream, "Warning! Valid cache consistency error, file '%s', url '%s'\n", c->File, c->Url);
			fprintf (stderr,    "Warning! Valid cache consistency error, file '%s', url '%s'\n", c->File, c->Url);
			debug             (("Warning! Valid cache consistency error, file '%s', url '%s'\n", c->File, c->Url));
		    }
		}
		else if (c->Flags & CACHE_QUEUED)
		{
		    if (c->File[0] == '@')
			fprintf (Fd, "%s q%s\n", c->File, c->Url);
		    else
		    {
			fprintf (LogStream, "Warning! Queued cache consistency error, file '%s', url '%s'\n", c->File, c->Url);
			fprintf (stderr,    "Warning! Queued cache consistency error, file '%s', url '%s'\n", c->File, c->Url);
			debug             (("Warning! Queued cache consistency error, file '%s', url '%s'\n", c->File, c->Url));
		    }
		}
		else
		{
		    fprintf (LogStream, "Warning! Unknown cache consistency error, file '%s', url '%s'\n", c->File, c->Url);
		    fprintf (stderr,    "Warning! Unknown cache consistency error, file '%s', url '%s'\n", c->File, c->Url);
		    debug             (("Warning! Unknown cache consistency error, file '%s', url '%s'\n", c->File, c->Url));
		}
	    }

    fclose (Fd);
}

/*)) */
/*(( "BuildCache ()" */

/* Scan cache directory, build up cache and delete invalid cache entries */

void BuildCache (void)
{
    DIR *Dir;
    cachearray_t *aa, *aaa;
    cache_t *c, *cc, *ccc;
    struct dirent *d;

    if (! (Dir = opendir (CURRENTDIR)))
    {
	fprintf (stderr, "%s: cannot open cache directory: %s\n", PrgName, strerror (errno));
	exit (20);
    }

    errno = 0;
    c = GetFreeCacheSlot ();
    c->File [0] = '\0';
    assert (c != NULL);

    while ( (d = readdir (Dir)) )
    {
	if (strlen (d->d_name) > MAX_FILENAME-1)
	{
	    if (d->d_name[0] != '.')                 /* no special file... */
		fprintf (stderr, "unknown file '%s' in cache directory\n", d->d_name);
	}
	else
	{
	    strcpy (c->File, d->d_name);
	    if (c->File [0] == '_')                  /* found cache data - look for cache url */
	    {
		debug (("found data '%s'\n", c->File));
		aa = Caches;
		for (cc = aa->Caches; ; cc++)
		{
		    if (cc >= & aa->Caches [MAX_CACHES])
		    {
			aa = aa->Next;
			assert (aa != NULL);
			cc = aa->Caches;
		    }
		    if (cc == c)
			break;
		    if (strcmp (&c->File[1], &cc->File[1]) == 0 && cc->File[0] == '@')
		    {
			cc->File[0] = '_';
			c->File[0]  = '@';
			ReadCacheUrl (cc, c->File);  /* already read... */
			c->File[0] = '\0';
			break;
		    }
		}
		if (c == cc)                         /* not found... */
		{
		    if (! (c = GetFreeCacheSlot ()) )/* keep it, may be we'll get the url file, too */
		    {
			fprintf (LogStream, "Cache table full while recacheing\n");
			debug (("Cache table full while recacheing\n"));
			break;                       /* break from while - no room left... */
		    }
		    c->File [0] = '\0';
		}
	    }
	    else if (c->File [0] == '@')             /* found cache url - look for cache data */
	    {
		debug (("found url '%s'\n", c->File));
		aa = Caches;
		for (cc = aa->Caches; ; cc++)
		{
		    if (cc >= & aa->Caches [MAX_CACHES])
		    {
			aa = aa->Next;
			assert (aa != NULL);
			cc = aa->Caches;
		    }
		    if (cc == c)
			break;
		    if (strcmp (&c->File[1], &cc->File[1]) == 0 && cc->File[0] == '_')
		    {
			ReadCacheUrl (cc, c->File);  /* already read... */
			c->File[0] = '\0';
			break;
		    }
		}
		if (c == cc)                         /* not found... */
		{
		    if (! (c = GetFreeCacheSlot ())) /* keep it, may be we'll get the data file, too */
		    {
			fprintf (LogStream, "Cache table full while recacheing\n");
			debug (("Cache table full while recacheing\n"));
			break;                       /* break from while - no room left... */
		    }
		    c->File [0] = '\0';
		}
	    }
	    else
	    {
		if (c->File [0] != '.')             /* .files, .httpproxy-log or standard directories (unix) */
		    fprintf (stderr, "unknown file '%s' in cache directory\n", c->File);
		c->File[0] = '\0';
	    }
	}
    }
    if (errno)
	fprintf (stderr, "directory error while building cache table, continueing: %s\n", strerror (errno));
    closedir (Dir);
    RemCacheEntry (c);

	/* now remove all incomplete cachentries (and files) */
    aa = Caches;
    for (cc = aa->Caches; ; cc++)
    {
	if (cc >= & aa->Caches [MAX_CACHES])
	{
	    aa = aa->Next;
	    assert (aa != NULL);
	    cc = aa->Caches;
	}
	if (cc == c)
	    break;
	if (cc->Url[0] == '\0' && cc->File [0] == '_')
	{
	    fprintf (LogStream, "cache file '%s' invalid, removing.\n", cc->File);
	    debug (("cache file '%s' invalid, removing.\n", cc->File));
	    RemCacheEntry (cc);
	}
    }

	/* mark all url only caches as queued */
    aa = Caches;
    for (cc = aa->Caches; ; cc++)
    {
	if (cc >= & aa->Caches [MAX_CACHES])
	{
	    aa = aa->Next;
	    assert (aa != NULL);
	    cc = aa->Caches;
	}
	if (cc == c)
	    break;
	if (cc->File [0] == '@')
	{
	    ReadCacheUrl (cc, cc->File);         /* the file name remains '@...' */
	    if (cc->File [0] == '@')
	    {
		cc->Flags = CACHE_QUEUED;        /* and ! CACHE_VAILD */
		aaa = Caches;
		for (ccc = aaa->Caches; ; ccc++)
		{
		    if (ccc >= & aaa->Caches [MAX_CACHES])
		    {
			aaa = aaa->Next;
			assert (aaa != NULL);
			ccc = aaa->Caches;
		    }
		    if (ccc == c)
			break;
		    if (ccc->File [0] == '_')
			if (cc != ccc && strcmp (ccc->Url, cc->Url) == 0)
			{
			    debug (("url '%s': queued url file %s associated with expired cache entry %s\n", cc->Url, cc->File, ccc->File));
			    assert (ccc->Flags & CACHE_VALID);
			    if (cc->File [9] != '@')           /* it has to be a special queued url file... */
			    {
				fprintf (LogStream, "Warning! Double cache consistency error, file '%s', url '%s', file2 '%s'\n", ccc->File, ccc->Url, cc->File);
				fprintf (stderr,    "Warning! Double cache consistency error, file '%s', url '%s', file2 '%s'\n", ccc->File, ccc->Url, cc->File);
				debug             (("Warning! Double cache consistency error, file '%s', url '%s', file2 '%s'\n", ccc->File, ccc->Url, cc->File));
			    }

			    ccc->Flags |= CACHE_QUEUED | CACHE_DELETETMP;
			    cc->File[0] = '\0';
			    RemCacheEntry (cc);
			    ccc = c;         /* break all */
			    break;
			}
		}
		if (ccc == c)
		    debug (("queued url '%s'\n", cc->Url));
	    }
	}
    }
}


/*)) */
/*(( "ScanCache()" */

/* Scan whether a specific cache is already there, filled and ready to serve */
/* Returns 0: no, not there  1: yes, all set up  -1: Failure
 * 2: error message in buffer */
/* Sets up the request struct for correct filling, opens all streams, etc. */

int ScanCache (request_t *Req, char *Url, int Known)
{
    int i;
    cache_t *c;
    cachearray_t *a;
    static time_t LastRequestTime;

    debug (("Searching for cache for Url '%s'\n", Url));
    LastRequestTime = ThisRequestTime;
    ThisRequestTime = time (NULL);
    debug (("Last: %ld, This: %ld, Diff: %ld, reload: %s\n", LastRequestTime, ThisRequestTime,
	    difftime (ThisRequestTime, LastRequestTime),
	    difftime (ThisRequestTime, LastRequestTime) < ReloadCacheTime ? "yes" : "no"));

    for (a = Caches; a; a = a->Next)
	for (c = a->Caches, i=0; i < MAX_CACHES; c++, i++)
	    if (c->Url[0])
	    {
		if (strcmp (c->Url, Url) == 0)
		{
		    if (c->Flags & CACHE_VALID)
		    {
			switch (CheckCacheTime (c, ExpireCacheTime)) {
			case 0:
			    if (c == LastCache && difftime (ThisRequestTime, LastRequestTime) < ReloadCacheTime)
			    {
				if (CheckCacheTime (c, 0) == -2)         /* force queueing / reloading */
				{
				    debug (("queued url on request\n"));
				    ErrToReq (Req, 203, NULL, "Your request for reloading the document is queued.<BR>\n"
					      "You will get the new document next time you are online.<BR>\n"
					      "An expired cache entry exists and can be viewed by immedeately reloading this document.");
				    return (2);
				}
				break;                                   /* forcing reload of url */
			    }

			    if (! (Req->Stream = fopen (c->File, "r")) )
			    {
				fprintf (LogStream, "cannot open cache file '%s', removing: %s\n", c->File, strerror (errno));
				debug (("cannot open cache file '%s', removing: %s\n", c->File, strerror (errno)));
				RemCacheEntry (c);
			    }
			    else
			    {
				debug (("Found cache entry, file '%s'\n", c->File));
				Req->Cache  = LastCache = c;             /* ok, found cache entry */
				Req->Flags |= REQ_DONE;
				return (1);
			    }
			    break;

				/* case -1: break; */
			case -2:
			    if (c == LastCache && difftime (ThisRequestTime, LastRequestTime) < ReloadCacheTime)
			    {
				debug (("Sending (invalid) cache file '%s' on request\n", c->File));
				if (! (Req->Stream = fopen (c->File, "r")) )
				{
				    fprintf (LogStream, "cannot open cache file '%s', removing: %s\n", c->File, strerror (errno));
				    debug (("cannot open cache file '%s', removing: %s\n", c->File, strerror (errno)));
				    RemCacheEntry (c);
				    break;
				}
				else
				{
				    Req->Cache  = c;                         /* ok, send invalid cache entry */
				    Req->Flags |= REQ_DONE;
				    return (1);
				}
			    }
			    else
			    {
				if (OffLine)
				{
				    debug (("Cache entry expired, queued\n"));
				    ErrToReq (Req, 203, NULL, "Your request is queued.<BR>\n"
					      "You will get the document next time you are online.<BR>\n"
					      "An expired cache entry exists and can be viewed by immedeately reloading this document.");
				    LastCache = c;
				    return (2);
				}
				else
				{
				    debug (("unqueueing and getting Cache entry\n"));
				    c->Flags   = 0;
				    c->File[0] = '_';
				    LastCache  = c;
				    goto HaveAlreadyCache;         /* sorry... */
				}
			    }
			}
		    }
		    else                                           /* c->Flags & CACHE_VALID */
		    {
			if (c->Flags & CACHE_QUEUED)
			{
			    if (OffLine)
			    {
				if (c == LastCache && difftime (ThisRequestTime, LastRequestTime) < ReloadCacheTime)
				{
				    errno = 0;
				    ErrToReq (Req, 404, "Already queued", "Your request is already queued.<BR>\n"
					      "You will get the document next time you are online.<BR>\n"
					      "You tried to get an expired cache entry, but this document is not cached right now.");
				    return (2);
				}
				else
				{
				    debug (("Cache entry already queued\n"));
				    errno = 0;
				    ErrToReq (Req, 203, NULL, "Your request is already queued.<BR>\n"
					      "You will get the document next time you are online.<BR>\n"
					      "There is no expired cache entry for this document.");
				    LastCache = c;
				    return (2);
				}
			    }
			    else
			    {
				debug (("unqueueing and getting Cache entry\n"));
				c->Flags   = 0;
				c->File[0] = '_';
				LastCache  = c;
				goto HaveAlreadyCache;              /* sorry... */
			    }
			}
			else
			{
			    debug (("Second request while getting URL\n"));
			    ErrToReq (Req, 204, NULL, "The same request was sent from another connection.\n"
				      "May be you requested a queued URL which is just being received.<BR>\n"
				      "Please try again in a few moments.");
			    LastCache = c;
			    return (2);
			}
		    }
		}
	    }

    if (! (Known || ProxyProxy || OffLine))      /* not cached, unknown and online without proxyproxy... don't know where to get it */
    {
	errno = 0;
	ErrToReq (Req, 404, "Unknown host", "You tried to get a document from an unknown host. Please check the URL.");
	debug (("#%02d: unknown host: URL '%s'\n", Req-Requests, Url));
	return (2);
    }


    if (! (c = GetFreeCacheSlot ()) )            /* not in cache so far */
    {
	if (OffLine)
	{
	    ErrToReq (Req, 500, "Cache table full", "I'm sorry, but there's no memory left for the cache table.<BR>"
		      "It is not possible to queue up your request.");
	    return (2);
	}
	else
	{
	    fprintf (LogStream, "Cache table full\n");
	    debug (("Cache table full\n"));
	    return (0);                          /* no more free entries - just proxy it */
	}
    }

    if (OffLine)
    {
	ErrToReq (Req, 203, NULL, "Your new request is queued.<BR>\n"
		  "You will get the document next time you are online.");
	debug (("Request queued.\n"));
	strcpy (c->Url, Url);
	c->Flags = CACHE_QUEUED;
	LastCache = c;
	SaveCacheUrl (c);
	c->File[0] = '@';
	return (2);
    }

HaveAlreadyCache:

    if (! (Req->Stream = fopen (c->File, "w")) )
    {
	fprintf (LogStream, "cannot open new cache file '%s': %s\n", c->File, strerror (errno));
	debug (("cannot open new cache file '%s': %s\n", c->File, strerror (errno)));
	c->File[0] = '\0';
	RemCacheEntry (c);
	return (0);                              /* just proxy it */
    }

    debug (("New cache file '%s'\n", c->File));
    strcpy (c->Url, Url);                        /* new cache entry */
    c->Flags = 0;
    Req->Cache = LastCache = c;
    return (0);
}


/*)) */
/*(( "GetCacheData ()" */

/* Fill request data space with data from cache
 * when a read error occures, no data will be received and ServWrite()
 * will automagically terminate the socket. */

void GetCacheData (request_t *Req)
{
    int Bytes;

    assert (Req->DataRecv < MAX_DATABUFFER);
    assert (Req->Stream);

    debug (("Getting more cache data - Url done: %s\n", Req->Flags & REQ_REQDONE ? "yes" : "no"));

    if ( (Bytes = fread (& Req->DataBuffer [Req->DataRecv], sizeof (char),
			MAX_DATABUFFER - Req->DataRecv, Req->Stream)) > 0)
	Req->DataRecv += Bytes;
    else
    if (! feof (Req->Stream))
    {
	fprintf (LogStream, "read error on cache file '%s': %s\n", Req->Cache->File, strerror (errno));
	debug (("read error on cache file '%s': %s\n", Req->Cache->File, strerror (errno)));
    }
    debug (("Read %d bytes\n", Bytes));
}


/*)) */
/*(( "BuildFdSets ()" */

/* Build fdsets (outstanding reads and writes) */

void BuildFdSets (fd_set *ReadSet, fd_set *WriteSet)
{
    int i;
    request_t *Req;

    FD_ZERO (ReadSet);
    FD_ZERO (WriteSet);

    debug (("Build: "));
    assert (RequestsFree >= 0);
    if (RequestsFree)
    {
	debug (("Server %d", (int) ServerSocket));
	FD_SET  (ServerSocket, ReadSet);
    }

    for (i=0, Req=Requests; i < MaxRequests; i++, Req++)
    {
	if (Req->Flags & REQ_REQSOCKET)
	{
	    assert (Req->ReqSocket != 0);
	    if (Req->ReqRecv < MAX_REQBUFFER-1)
	    {
		debug ((", Read Req %d", Req-Requests));
		FD_SET (Req->ReqSocket, ReadSet);
	    }
	    if (Req->DataRecv > Req->DataSent)
	    {
		debug ((", Write Req %d", Req-Requests));
		FD_SET (Req->ReqSocket, WriteSet);
	    }
	}
	if (Req->Flags & REQ_CONNSOCKET)
	{
	    assert (Req->ConnSocket != 0);
	    if (Req->ReqRecv > Req->ReqSent || Req->UrlRecv > Req->UrlSent) /* Waiting for connect or sending request */
	    {
		debug ((", Write Con %d", Req-Requests));
		FD_SET (Req->ConnSocket, WriteSet);
	    }
	    if (Req->DataRecv < MAX_DATABUFFER)                /* Transfer Data */
	    {
		debug ((", Read Con %d", Req-Requests));
		FD_SET (Req->ConnSocket, ReadSet);
	    }
	}
    }
    debug (("\n"));
}


/*)) */
/*(( "CreateUrlRequest()/CheckUrl()" */

/* Create Url request according to specification */
/* Returns the length of the request. */
/* Prot and Host may be NULL, in that case Name is to be asumed to be a full URL.
 * No checks for Proxyproxy mode are done in this case. */

size_t CreateUrlRequest (const char *Method, const char *Prot, const char *Host, int Port, const char *Name, char *ReqSpace, int HttpOne)
{
    char Buffer [MAX_URLBUFFER];
    char *p, *DokPrint;

    if (Prot && Host)
    {
	if (ProxyProxy)
	{
	    if (Port > -1)
		sprintf (ReqSpace, "%s %s://%s:%d/", Method, Prot, Host, Port);
	    else
		sprintf (ReqSpace, "%s %s://%s/", Method, Prot, Host);
	}
	else
	    sprintf (ReqSpace, "%s /", Method);
    }
    else
	sprintf (ReqSpace, "%s ", Method);

    DokPrint = ReqSpace + strlen (ReqSpace);

    for (p = Buffer; *Name; )
	if (isvalidhttp (*Name))
	    *p++ = *Name++;
	else
	{
	    sprintf (p, "%%%02x", *Name++);
	    p += 3;
	}
    *p = '\0';

    if (HttpOne)
	sprintf (DokPrint, "%s HTTP/1.0\r\n", Buffer);
    else
	sprintf (DokPrint, "%s\r\n", Buffer);
    debug (("Request '%s', len %d created.\n", ReqSpace, strlen (ReqSpace)));
    return (strlen (ReqSpace));
}


/* Check whether the URL is read completely. Sets REQ_REQDONE accordingly.
 * Attention! This routine relys on the fact, that ScanUrl was called already!
 * That means especially, that the first line was already read completely. */
/* The routine does not check for Url termination on Buffer boundaries. So
 * the ReqBuffer has to be shifted always (four bytes remaining always) and
 * never be cleared at all. */

void CheckUrl (request_t *Req)
{
    debug (("Checking Url\n"));
    if (Req->Flags & REQ_HTTP1X0)
    {
	if (strstr (Req->ReqBuffer, "\n\n") || strstr (Req->ReqBuffer, "\r\r") ||
	    strstr (Req->ReqBuffer, "\n\r\n\r") || strstr (Req->ReqBuffer, "\r\n\r\n"))
	    Req->Flags |= REQ_REQDONE;
    }
    else
	Req->Flags |= REQ_REQDONE;
#ifdef DEBUG
   if (Req->Flags & REQ_REQDONE)
	debug (("http request complete\n"));
#endif
}


/*)) */
/*(( "ScanUrl ()" */

/* Scan the URL and divide it into several parts. Understands http/0.9
 * and http/1.0 versions right now. Sets REQ_REQDONE when URL is complete. */
/* Req->UrlBuffer is valid after calling this routine and REQ_URLDONE set. */
/* Returns -1 in case of failure, 0 on correct termination or found cache,
 * 1: found cache slot, FileD is open, 2: failure, cache contains error */
/* Port and Address are value-return arguments. */

int ScanUrl (request_t *Req, struct in_addr *Address, int *Port)
{
    struct hostent *HostEnt;
    struct in_addr In;
    char BufMETH [12];
    char BufPROT [16];
    char BufDOK [MAX_URLSAVE + 256];                                       /* a critical array... but that's enough */
    char BufURL [128];
    char BufVERS [8];
    int  CharsRead, CharsRead2, FoundHost = 0, i;
    char *DokPtr, *PortPtr;
    char *Host, *ObjectName;

    if (sscanf (Req->ReqBuffer, "%11s %15[a-zA-Z0-9]%n", BufMETH, BufPROT, &CharsRead) < 2)
	return (-1);
    debug (("METHOD '%s', PROT '%s'\n", BufMETH, BufPROT));
    if (CharsRead >= Req->ReqRecv)
	return (-1);
    if (OffLine && strcasecmp (BufMETH, "get") != 0)                       /* no get command */
	return (-1);
    if (! ProxyProxy)
	if (strcasecmp (BufPROT, "http") != 0)                             /* we only support http urls so far */
	    return (-1);
    if (strncmp (& Req->ReqBuffer [CharsRead], "://", 3) != 0)
	return (-1);
    if (sscanf (& Req->ReqBuffer [CharsRead+3], "%126s%n", BufURL, &CharsRead2) < 1)
	return (-1);
    debug (("URL '%s'\n", BufURL));
    CharsRead += 3 + CharsRead2;

    if (! (DokPtr = strchr (BufURL, '/')) )                                /* no host... */
	return (-1);
    *DokPtr++ = 0;
    ObjectName = DokPtr;

    if (strcasecmp (BufPROT, "http") != 0)
	*Port = -1;
    else
	*Port = StdHttpPort;
    if ( (PortPtr = strchr (BufURL, ':')) )
    {
	*PortPtr = 0;
	*Port = atoi (PortPtr + 1);
    }

    Host = BufURL;
    if (! (ProxyProxy || OffLine))
    {
	if ( (In.s_addr = inet_addr (BufURL)) != -1)
	    FoundHost = 1;
	else if ( (HostEnt = gethostbyname (BufURL)) ) 
	{
	    debug (("gethostbyname () succeded\n"));
	    memmove (&In, HostEnt->h_addr, sizeof (struct in_addr));
	    FoundHost = 1;
	}
    }
    else
	FoundHost = 1;           /* we do know our proxy host */

    *Address = In;

    BufVERS [2] = 0;                                                       /* BufVERS is missused here... */
    if (*Port > -1)
	sprintf (BufDOK, "%s://%s:%d/", BufPROT, Host, *Port);
    else
	sprintf (BufDOK, "%s://%s/", BufPROT, Host);
    for (PortPtr = BufDOK + strlen (BufDOK); *DokPtr; DokPtr++, PortPtr++)
    {
	if (*DokPtr == '%')
	{
	    if (! (BufVERS [0] = *++DokPtr) )
		break;
	    if (! (BufVERS [1] = *++DokPtr) )
		break;
	    if ( (i = strtol (BufVERS, NULL, 0x10)) )
		*PortPtr = i;
	    else
		*PortPtr = '?';
	}
	else
	    *PortPtr = *DokPtr;
    }
    *PortPtr = 0;

    debug (("Proto '%s', Host '%s', Port %d, Dok '%s'\n", BufPROT, Host, *Port, BufDOK));

    if (Req->ReqBuffer [CharsRead] == ' ' && CharsRead < Req->ReqRecv)
	if (sscanf (& Req->ReqBuffer [CharsRead], " %5s", BufVERS) == 1)
	    if (strcasecmp (BufVERS, "http/") == 0)
	    {
		debug (("got HTTP/1.0 or greater request\n"));
		Req->Flags |= REQ_HTTP1X0;
	    }

    CheckUrl (Req);
    if (strlen (BufDOK) > MAX_URLSAVE-1)                     /* That one cannot be cached */
    {
	fprintf (LogStream, "ReqBuffer size (%d chars) exceeded - document '%s' is not cached\n", MAX_URLSAVE-1, BufDOK);
	debug (("ReqBuffer size (%d chars) exceeded - document '%s' is not cached\n", MAX_URLSAVE-1, BufDOK));
	return (0);
    }

	/* create UrlBuffer entry */
    assert (Req->UrlSent == 0);
    Req->UrlRecv = CreateUrlRequest (BufMETH, BufPROT, BufURL, *Port, ObjectName, Req->UrlBuffer, Req->Flags & REQ_HTTP1X0);
    PortPtr = strchr (& Req->ReqBuffer [CharsRead], '\n');   /* PortPtr und DokPtr are missused here */
    DokPtr  = strchr (& Req->ReqBuffer [CharsRead], '\r');
    assert (DokPtr == NULL || PortPtr == NULL || PortPtr == DokPtr + 1 || DokPtr == PortPtr + 1);
    if (DokPtr == PortPtr + 1 || PortPtr == NULL)
	PortPtr = DokPtr;
    assert (PortPtr != NULL);

    Req->ReqSent = PortPtr - Req->ReqBuffer + 1;
    Req->Flags  |= REQ_URLDONE;

    if (strcasecmp (BufMETH, "get") != 0)                                  /* only get's will be cached */
    {
	debug (("no get method - no caching\n"));
	return (0);
    }

    return (ScanCache (Req, BufDOK, FoundHost));
}


/*)) */
/*(( "DeleteRequest/Connect ()" */

/* Close the connection socket only. When a request socket is still open
 * and no data is in the cache, close all.
 * Delete cache entry in case of an error. Check the answer whether it is
 * an error message or a regular answer. Errors should not be cached! */
/* When Flags == 0 nothing is done at all */

void DeleteConnect (request_t *Req, int ok)
{
/*    cache_t tmp;*/

    debug (("DeleteCon %d\n", Req-Requests));
    if (! Req->Flags)
	return;
    if (Req->Flags & REQ_CONNSOCKET)
    {
	CloseSocket (Req->ConnSocket);
	Req->Flags &= ~REQ_CONNSOCKET;
	if (Req->Stream)
	    fclose (Req->Stream);
	Req->Stream = NULL;
	if (ok)
	{
	    if (Req->Cache && ! (Req->Cache->Flags & CACHE_QUEUED)) /* we've written into a cache file */
	    {
		SaveCacheUrl (Req->Cache);
		Req->Cache = NULL;
	    }
	}
	else if (Req->Cache)                                        /* not ok - a error occured */
	{
	    fprintf (LogStream, "error while receiving URL '%s' - removing\n", Req->Cache->Url);
	    debug (("error while receiving URL '%s' - removing\n", Req->Cache->Url));
	    Req->Cache->Url[0] = '\0';
	    RemCacheEntry (Req->Cache);
	    Req->Cache = NULL;
	}
    }
    if (Req->Flags & REQ_REQSOCKET)
    {
	if (Req->DataRecv <= Req->DataSent && (Req->Flags & REQ_REQDONE))
	{
	    CloseSocket (Req->ReqSocket);       /* All done. Hugh! */
	    Req->Flags = 0;
	}
	else
	    Req->Flags |= REQ_DONE;
    }
    else
	Req->Flags = 0;
    if (! Req->Flags)
	RequestsFree++;
}


/* Close the request socket only. When a connection socket is already up
 * (and enough data is already read) continue filling up the cache. When
 * everything is done, close all. The URL should be checked, too.
 * When it is not complete, all connections should be terminated
 * on CacheUnreadRequests false. */
/* Don't call DeleteConnect when Flags == 0... */

void DeleteRequest (request_t *Req, int ok)
{
    debug (("DeleteReq %d\n", Req-Requests));
    if (Req->Flags & REQ_REQSOCKET)
    {
	CloseSocket (Req->ReqSocket);
	Req->Flags &= ~REQ_REQSOCKET;
    }

    if (Req->Flags & REQ_DONE)                  /* Maybe we're getting data from the cache */
    {
	if (Req->Stream)
	    fclose (Req->Stream);
	Req->Stream = NULL;
	DeleteConnect (Req, ok);
	Req->Flags  = 0;                        /* All done. Hugh! */
	debug (("All done - deleterequest\n"));
    }
    else if (Req->Flags & REQ_REQDONE)
    {
	if (! (Req->Flags & REQ_CONNSOCKET))    /* Aborting cache sending */
	    DeleteConnect (Req, FALSE);
	else if (CacheUnreadRequests)
	    Req->DataSent = Req->DataRecv = 0;  /* Continue getting data into the cache */
    }
    else
    {
	Req->Flags |= REQ_DONE;
	DeleteConnect (Req, FALSE);             /* No request known so far... */
    }
}


/*)) */
/*(( "ServConnect ()" */

/* The first line of the URL is already there, so we can connect to the
 * remote host. */

void ServConnect (request_t *Req)
{
    struct sockaddr_in SockIn;
    struct sockaddr_in *SockPtr = &SockIn;
    ioctl_t on = 1;
    int  Port;

    debug (("ServCon for Req %d; Flags %x\n", Req-Requests, Req->Flags));

    if (Req->Flags & REQ_REQSOCKET)
    {
	if (strchr (Req->ReqBuffer, '\n') == 0 &&
	    strchr (Req->ReqBuffer, '\r') == 0)         /* first line of URL is not there... */
	    return;

	if (Req->Flags & REQ_URLDONE)  /* already set up */
	{
	    CheckUrl (Req);
	    return;
	}
    }

    memset ((char *) &SockIn, 0, sizeof (struct sockaddr_in));

    switch (ScanUrl (Req, &SockIn.sin_addr, &Port)) {
    case 0:                                   /* need to get document from remote host */
	if (Req->Cache)
	{
	    fprintf (LogStream, "#%02d: getting URL '%s'\n", Req-Requests, Req->Cache->Url);
	    debug (("#%02d: getting URL '%s'\n", Req-Requests, Req->Cache->Url));
	}
	else
	{
	    fprintf (LogStream, "#%02d: serving uncacheable request '%s'\n",
		     Req-Requests, Req->ReqBuffer);
	    debug  (("#%02d: serving uncacheable request '%s'\n",
		     Req-Requests, Req->ReqBuffer));
	}
	break;

    case 1:                                   /* already cached - no need to connect to remote host */
	assert (Req->Cache != NULL);
	fprintf (LogStream, "#%02d: sending URL '%s' from cache '%s'\n",
		 Req-Requests, Req->Cache->Url, Req->Cache->File);
	debug  (("#%02d: sending URL '%s' from cache '%s'\n",
		 Req-Requests, Req->Cache->Url, Req->Cache->File));
	GetCacheData (Req);
	return;

    case 2:                                   /* send error message from buffer */
	return;

    default:
	{
	    fprintf (LogStream, "#%02d: invalid request\n", Req-Requests);
	    debug (("#%02d: invalid request\n", Req-Requests));
	    Req->Flags |= REQ_DONE | REQ_REQDONE;
	    errno = 0;
	    ErrToReq (Req, 500, "Invalid Request", "Your Request is not a valid URL.<BR>\n"
		      "Please check it and try again.");
	    return;
	}
    }

    if (OffLine)
    {
	errno = 0;
	ErrToReq (Req, 500, "Unable to serv", "It is not possible to connect to a remote host in offline mode.<BR>\n"
		  "Either the requested URL is to long to be queued or an internal error has occured. When the\n"
		  "URL is rather short, please contact the author of" PRG_NAME);
	return;
    }

    if (ProxyProxy)
	SockPtr = &ProxyProxyIn;
    else
    {
	SockIn.sin_family     = AF_INET;
	SockIn.sin_port       = htons (Port);
    }


    if ( (Req->ConnSocket = socket (AF_INET, SOCK_STREAM, 0)) < 0)                 /* open connection */
    {
	syslog (LOG_ERR, "%s: socket () failed", PrgName);
	DeleteRequest (Req, FALSE);
	return;
    }

    Req->Flags |= REQ_CONNSOCKET;

#ifdef FIOASYNC
    if (ioctl (Req->ConnSocket, FIOASYNC, (caddr_t) &on) < 0)
    {
	syslog (LOG_ERR, "%s: ioctl() failed for FIOASYNC: %s", PrgName, strerror (errno));
	DeleteRequest (Req, FALSE);
	return;
    }
#endif
#ifdef FIONBIO
    if (ioctl (Req->ConnSocket, FIONBIO, (caddr_t) &on) < 0)
    {
	syslog (LOG_ERR, "%s: ioctl() failed for FIONBIO: %s", PrgName, strerror (errno));
	DeleteRequest (Req, FALSE);
	return;
    }
#endif

    if (connect (Req->ConnSocket, (struct sockaddr *) SockPtr, sizeof (struct sockaddr_in)) < 0)
    {
	if (errno != EINPROGRESS)
	{
	    if (ProxyProxy)
		ErrToReq (Req, 500, "Proxyproxy Host Unreachable",
			  "The host you specified with the 'proxy' option can't be contacted.<BR>\n"
			  "Please wait until the proxy is up again or start " PRG_NAME " in offline mode.");
	    else
		ErrToReq (Req, 500, "Host Unreachable",
			  "The specified host can't be contacted.<BR>\n"
			  "Please wait until it is up again or try any other URLs.");
	    return;
	}
    }
    debug (("ServCon -> Conn %d\n", (int) Req->ConnSocket));
}


/*)) */
/*(( "ServServer ()" */

/* Serv the server port. Accept new connections, set up request database */

void ServServer (void)
{
    struct sockaddr_in PeerIn;
    len_t  Len = sizeof (struct sockaddr_in);
    request_t *Req = Requests;
    ioctl_t on = 1;

    assert (RequestsFree > 0);

    while (Req->Flags)                        /* any Flags set -> occupied */
	Req++;
    debug (("accept Req %d\n", Req-Requests));
    assert (Req - Requests < MaxRequests);

    if ( (Req->ReqSocket = accept (ServerSocket, (struct sockaddr *) &PeerIn, &Len)) < 0)
    {
	syslog (LOG_ERR, "%s: accept() failed: %s", PrgName, strerror (errno));
	return;
    }

    fprintf (LogStream, "#%02d: new request from %s:%d\n",
	     Req-Requests, inet_ntoa (PeerIn.sin_addr), PeerIn.sin_port);
    debug  (("#%02d: new request from %s:%d\n",
	     Req-Requests, inet_ntoa (PeerIn.sin_addr), PeerIn.sin_port));

    Req->Flags |= REQ_REQSOCKET;
    Req->ReqBuffer[0] = Req->UrlBuffer [0] = Req->DataBuffer [0] = '\0';
    Req->ReqSent = Req->ReqRecv = Req->UrlSent = Req->UrlRecv =
		   Req->DataSent = Req->DataRecv = 0;
    Req->Cache = NULL;
    Req->Stream = NULL;
    RequestsFree--;

#ifdef FIOASYNC
    if (ioctl (Req->ReqSocket, FIOASYNC, (caddr_t) &on) < 0)
    {
	syslog (LOG_ERR, "%s: ioctl() failed for FIOASYNC: %s", PrgName, strerror (errno));
	DeleteRequest (Req, FALSE);
	return;
    }
#endif
#ifdef FIONBIO
    if (ioctl (Req->ReqSocket, FIONBIO, (caddr_t) &on) < 0)
    {
	syslog (LOG_ERR, "%s: ioctl() failed for FIONBIO: %s", PrgName, strerror (errno));
	DeleteRequest (Req, FALSE);
	return;
    }
#endif
}


/*)) */
/*(( "ServRead ()" */

/* Server for all reads */

void ServRead (fd_set *ReadSet)
{
    int i, Bytes;
    register request_t *Req;

    if (FD_ISSET (ServerSocket, ReadSet))
	ServServer ();
    for (i=0, Req=Requests; i < MaxRequests; i++, Req++)
    {
	if ((Req->Flags & REQ_REQSOCKET) && FD_ISSET (Req->ReqSocket, ReadSet))
	{
	    assert (Req->ReqRecv < MAX_REQBUFFER-1);
	    if ( (Bytes = recv (Req->ReqSocket, & Req->ReqBuffer [Req->ReqRecv],
				(long) MAX_REQBUFFER-1 - Req->ReqRecv, 0)) < 0)
	    {
		fprintf (LogStream, "#%02d: on receiving URL: %s\n", Req-Requests, strerror (errno));
		debug (("#%02d: on receiving URL: %s\n", Req-Requests, strerror (errno)));
		DeleteRequest (Req, FALSE);
	    }
	    else
	    {
		Req->ReqRecv += Bytes;
		Req->ReqBuffer [Req->ReqRecv] = '\0';        /* needed for strchr() */

		debug (("Read %d bytes from Req %d to 0x%x containing:\n'%s'\n", Bytes,
			Req-Requests, Req->ReqRecv-Bytes, & Req->ReqBuffer [Req->ReqRecv - Bytes]));
#if defined (DEBUG) && ! defined (_M68020)
		if ((Req->ReqRecv - Bytes) & 0x0001)
		    debug (("(odd address...\n"));
		else
#endif
		debug (("= 0x %08lx %08lx %08lx %08lx\n", * (long *) (Req->ReqBuffer + Req->ReqRecv - Bytes),
			* (long *) (Req->ReqBuffer + Req->ReqRecv - Bytes + 4), * (long *) (Req->ReqBuffer + Req->ReqRecv - Bytes + 8),
			* (long *) (Req->ReqBuffer + Req->ReqRecv - Bytes + 12)));

		if (Bytes == 0)                              /* request socket gone away */
		{
		    DeleteRequest (Req, TRUE);
		    continue;
		}
		ServConnect (Req);                           /* Check if anything can be done already */
		if ((Req->Flags & REQ_DONE) && (Req->Flags & REQ_REQDONE) && (Req->DataRecv == 0))
		{
		    DeleteRequest (Req, TRUE);               /* all done, another time */
		    continue;
		}
		if ((Req->ReqRecv == MAX_REQBUFFER-1) && ! (Req->Flags & REQ_CONNSOCKET))
		{                                            /* buffer full and no connection yet */
		    if (Req->Flags & REQ_URLDONE)
			Req->ReqRecv = Req->ReqSent = 0;     /* Skip remaining data (we won't send it...) */
		    else
		    {
			fprintf (LogStream, "#%02d: URL buffer overflow\n", Req-Requests);
			debug (("#%02d: URL buffer overflow\n", Req-Requests));
			DeleteRequest (Req, FALSE);
		    }
		}
	    }
	}
	if ((Req->Flags & REQ_CONNSOCKET) && FD_ISSET (Req->ConnSocket, ReadSet))
	{
	    assert (Req->DataRecv < MAX_DATABUFFER);
	    if ( (Bytes = recv (Req->ConnSocket, & Req->DataBuffer [Req->DataRecv],
				(long) MAX_DATABUFFER - Req->DataRecv, 0)) < 0)
	    {
		if (ProxyProxy)
		    ErrToReq (Req, 500, "Proxyproxy Host Unreachable / read() failed",
			      "The host you specified with the 'proxy' option can't be contacted.<BR>\n"
			      "Please wait until the proxy is up again or start " PRG_NAME " in offline mode.");
		else
		    ErrToReq (Req, 500, "Host Unreachable / read() failed",
			      "The specified host can't be contacted.<BR>\n"
			      "Please wait until it is up again or try any other URLs.");
		DeleteConnect (Req, FALSE);
	    }
	    else
	    {
		debug (("Read %d bytes from Con %d\n", Bytes, Req-Requests));
		if (Bytes == 0)                               /* We're done */
		{
		    DeleteConnect (Req, TRUE);
		    continue;
		}
		else if (Req->Stream)
		    fwrite (& Req->DataBuffer [Req->DataRecv], sizeof (char),      /* no error checking... */
			    Bytes, Req->Stream);
		if (Req->Flags & REQ_REQSOCKET)
		    Req->DataRecv += Bytes;                   /* else: only save cache data */
	    }
	}
    }
}


/*)) */
/*(( "ServWrite ()" */

/* Server for all writes */

void ServWrite (fd_set *WriteSet)
{
    int i, Bytes;
    register request_t *Req;

    for (i=0, Req=Requests; i < MaxRequests; i++, Req++)
    {
	if ((Req->Flags & REQ_REQSOCKET) && FD_ISSET (Req->ReqSocket, WriteSet))
	{
	    assert (Req->DataRecv > Req->DataSent);
	    if ( (Bytes = send (Req->ReqSocket, & Req->DataBuffer [Req->DataSent],
				(long) Req->DataRecv - Req->DataSent, 0)) < 0)
	    {
		fprintf (LogStream, "#%02d: on sending data: %s\n", Req-Requests, strerror (errno));
		debug (("#%02d: on sending data: %s\n", Req-Requests, strerror (errno)));
		DeleteRequest (Req, FALSE);
		continue;
	    }
	    else
	    {
		debug (("Wrote %d bytes to Req %d\n", Bytes, Req-Requests));
		Req->DataSent += Bytes;
		assert (Req->DataRecv >= Req->DataSent);
		if (Bytes == 0)                            /* request socket is unable to receive data */
		{
		    fprintf (LogStream, "#%02d: request socket unable to receive data\n", Req-Requests);
		    debug (("#%02d: request socket unable to receive data\n", Req-Requests));
		    DeleteRequest (Req, FALSE);
		    continue;
		}

		if (Req->DataSent == Req->DataRecv)        /* clear / shift data buffer */
		    Req->DataSent = Req->DataRecv = 0;
		else if (Req->DataSent > SHIFT_DATABUFFER)
		{
		    debug (("shifting databuffer size %d by %d bytes\n", Req->DataRecv - Req->DataSent, Req->DataSent));
		    memmove (Req->DataBuffer, & Req->DataBuffer [Req->DataSent],
			     Req->DataRecv - Req->DataSent);
		    Req->DataRecv -= Req->DataSent;
		    Req->DataSent = 0;
		}
		if (Req->DataRecv < MAX_DATABUFFER && (Req->Flags & REQ_DONE) && Req->Stream)
		    GetCacheData (Req);
		if (Req->DataRecv == 0 && (Req->Flags & REQ_DONE) && (Req->Flags & REQ_REQDONE))   /* We're already done */
		    DeleteRequest (Req, TRUE);

	    }
	}
	if ((Req->Flags & REQ_CONNSOCKET) && FD_ISSET (Req->ConnSocket, WriteSet))
	{
	    if (Req->UrlRecv > Req->UrlSent)
		Bytes = send (Req->ConnSocket, &Req->UrlBuffer [Req->UrlSent],
			      (long) Req->UrlRecv - Req->UrlSent, 0);
	    else
	    {
		assert (Req->ReqRecv > Req->ReqSent);
		Bytes = send (Req->ConnSocket, & Req->ReqBuffer [Req->ReqSent],
			      (long) Req->ReqRecv - Req->ReqSent, 0);
	    }
	    if (Bytes < 0)
	    {
		if (ProxyProxy)
		{
		    fprintf (LogStream, "#%02d: proxyproxy host unreachable: %s\n", Req-Requests, strerror (errno));
		    debug (("#%02d: proxyproxy host unreachable: %s\n", Req-Requests, strerror (errno)));
		}
		else
		{
		    fprintf (LogStream, "#%02d: on sending url: %s\n", Req-Requests, strerror (errno));
		    debug (("#%02d: on sending url: %s\n", Req-Requests, strerror (errno)));
		}
		DeleteConnect (Req, FALSE);
	    }
	    else
	    {
		debug (("Wrote %d bytes to Conn %d from %s, 0x%x\n", Bytes, Req-Requests,
			Req->UrlRecv > Req->UrlSent ? "UrlBuffer" : "ReqBuffer", Req->UrlRecv > Req->UrlSent ? Req->UrlSent : Req->ReqSent));

		if (Req->UrlRecv > Req->UrlSent)
		    Req->UrlSent += Bytes;
		else
		    Req->ReqSent += Bytes;
		if (Bytes == 0)                               /* connection socket is unable to receive url */
		{
		    fprintf (LogStream, "#%02d: connection socket unable to receive url\n", Req-Requests);
		    debug (("#%02d: connection socket unable to receive url\n", Req-Requests));
		    DeleteConnect (Req, FALSE);
		    continue;
		}

		if (Req->ReqSent > SHIFT_REQBUFFER)
		{
		    debug (("shifting reqbuffer size %d+4 by %d bytes\n", Req->ReqRecv - Req->ReqSent, Req->ReqSent-4));
		    memmove (Req->ReqBuffer, & Req->ReqBuffer [Req->ReqSent - 4],
			     Req->ReqRecv - Req->ReqSent + 4);
		    Req->ReqRecv -= Req->ReqSent - 4;
		    Req->ReqSent = 4;
		}
	    }
	}
    }
}


/*)) */
/*(( "CacheInUse()/RequestQueued()" */

/* Check wheather a cache entry is used in any other request */

request_t *CacheInUse (cache_t *c)
{
    request_t *Req = Requests;
    int i;

    for (i = 0; i < MaxRequests; i++, Req++)
	if (Req->Cache == c)
	    return (Req);
    return (NULL);
}


/* Initiate contact to remote host for a queued request */

void RequestQueued (cache_t *c)
{
    request_t *Req = Requests;
    int     Len;

    assert (MAX_REQBUFFER >= 4 * MAX_URLSAVE);  /* to be on the save side */
    fprintf (LogStream, "initiating request for Url '%s'\n", c->Url);

    while (Req->Flags)                        /* any Flags set -> occupied */
	Req++;
    assert (Req - Requests < MaxRequests);

    Req->Flags  = REQ_REQDONE;
    Req->ReqSent= Req->DataSent = Req->DataRecv = Req->UrlSent = Req->UrlRecv = 0;
    Req->Cache  = NULL;
    Req->Stream = NULL;
    RequestsFree--;

    Len = CreateUrlRequest ("GET", NULL, NULL, 0, c->Url, Req->ReqBuffer, TRUE);
    sprintf (& Req->ReqBuffer [Len], "User-Agent: %s\r\nAccept: */*\r\n\r\n", VERSHTTP);
    Req->ReqRecv = strlen (Req->ReqBuffer);
    debug (("initiating request '%s'\n", Req->ReqBuffer));
    if (c->File [0] == '@')
	c->Url [0] = '\0';
    RemCacheEntry (c);
    ServConnect (Req);
}


/*)) */
/*(( "CheckGetQueued()" */

/* Check number of pending requests and initiate getting of queued URLs */

void CheckGetQueued (void)
{
    static cachearray_t *a = NULL;
    static int Nr = 0;
    static int State = 1;
    cache_t *c;

    if (State == 3 || ! GetQueued)
	return;
    if (a == NULL)
	a = Caches;

    debug (("State %d\n", State));

    while (RequestsFree > MIN_REQUESTS)
	switch (State) {
	case 1:                         /* get all CACHE_QUEUED & ! CACHE_VALID */
	    for (c = & a->Caches [Nr]; Nr < MAX_CACHES; Nr++, c++)
		if ((c->Flags & CACHE_QUEUED) && ! (c->Flags & CACHE_VALID))    /* get urls without expired caches first */
		    if (! CacheInUse (c))             /* inUse caches won't be get at all... */
		    {
			RequestQueued (c);
			break;
		    }
	    if (Nr < MAX_CACHES)
		continue;
	    Nr = 0;
	    if (a->Next)
	    {
		a = a->Next;
		continue;
	    }

	    State = 2;
	    a     = Caches;  /* no break! */

	case 2:
	    for (c = & a->Caches [Nr]; Nr < MAX_CACHES; Nr++, c++)
		if (c->Flags & CACHE_QUEUED)
		{
		    assert (c->Flags & CACHE_VALID);
		    if (! CacheInUse (c))
		    {
			RequestQueued (c);
			break;
		    }
		}
	    if (Nr < MAX_CACHES)
		continue;
	    Nr = 0;
	    if (a->Next)
	    {
		a = a->Next;
		continue;
	    }
	    State = 3;
	    return;

	default:
	    assert (0);
	}
}


/*)) */
/*(( "DeleteInvalidCaches()" */

/* Shutdown: Close and delete all invalid cache entries. */

void DeleteInvalidCaches (void)
{
    cachearray_t *a;
    cache_t *c, Tmp, *cc;
    request_t *Req;
    int     i;

    for (a = Caches; a; a = a->Next)
	for (i = 0, c = a->Caches; i < MAX_CACHES; i++, c++)
	    if (c->File[0] && (c->Flags & (CACHE_VALID | CACHE_QUEUED)) +0 == 0)      /* !!! */
	    {
		Req = CacheInUse (c);
		if (Req)
		{
		    if (Req->Stream)                      /* close cache stream */
			fclose (Req->Stream);
		    Req->Stream = NULL;
		    if (c->Url[0])
		    {
			fprintf (LogStream, "removing and queueing URL '%s'\n", c->Url);
			SaveCacheUrl (c);
			assert (c->Url[0] != '\0');
			Tmp = *c;
			c->Url[0] = '\0';                 /* keep the Urlfile */
			RemCacheEntry (c);
			cc = GetFreeCacheSlot ();
			assert (cc != NULL);
			InitCacheSlot (cc, &Tmp, '@', '.');
			cc->Flags = CACHE_QUEUED;
		    }
		    else
			RemCacheEntry (c);
		}
		else
		    debug (("File '%s', Url '%s', Flags 0x%x without Request!\n", c->File, c->Url, c->Flags));
	    }
}


/*)) */
/*(( "main ()" */

/* The main routine */

void main (int argc, char **argv)
{
    fd_set ReadSet;
    fd_set WriteSet;
    char   *ProxyHost = NULL;
    char   *LogFile   = "." PRG_NAME "-log";
    int    ProxyPort;

    PrgName = *argv++;

    if (--argc == 1)
	if (**argv == '?')
	{
	    fprintf (stderr, "Usage: %s [proxy PROXYHOST PROXYPORT] [port PORT] [cache DIR] [del SECONDS]\n"
		     "[expire SECONDS] [reload SECONDS] [log FILE] [numreq NUMBER]\n"
		     "[unread] [offline] [get]\n"
			     "The cache keyword will change the local directory.\n", PrgName);
	    exit (0);
	}

    while (argc)
    {
	if (strcasecmp (*argv, "proxy") == 0)
	{
	    if (argc < 3)
	    {
		fprintf (stderr, "%s: need two arguments for 'proxy'\n", PrgName);
		exit (1);
	    }
	    argv++;
	    argc -= 3;
	    ProxyHost = *argv++;
	    if ( (ProxyPort = atoi (*argv++)) <= 0)
	    {
		fprintf (stderr, "%s: Wrong second argument for 'proxy' (need the port number).\n", PrgName);
		exit (1);
	    }
	}
	else if (strcasecmp (*argv, "port") == 0)
	{
	    if (argc < 2)
	    {
		fprintf (stderr, "%s: need a argument for 'port'\n", PrgName);
		exit (1);
	    }
	    argv++;
	    argc -= 2;
	    ServerPort = atoi (*argv++);
	}
	else if (strcasecmp (*argv, "cache") == 0)
	{
	    if (argc < 2)
	    {
		fprintf (stderr, "%s: need a argument for 'cache'\n", PrgName);
		exit (1);
	    }
	    argv++;
	    argc -= 2;
	    if (chdir (*argv++))
	    {
		fprintf (stderr, "%s: no directory '%s': %s\n", PrgName, argv[-1], strerror (errno));
		exit (1);
	    }
	}
	else if (strcasecmp (*argv, "del") == 0)
	{
	    if (argc < 2)
	    {
		fprintf (stderr, "%s: need a argument for 'del'\n", PrgName);
		exit (1);
	    }
	    argv++;
	    argc -= 2;
	    DelCacheTime = atoi (*argv++);
	}
	else if (strcasecmp (*argv, "expire") == 0)
	{
	    if (argc < 2)
	    {
		fprintf (stderr, "%s: need a argument for 'expire'\n", PrgName);
		exit (1);
	    }
	    argv++;
	    argc -= 2;
	    ExpireCacheTime = atoi (*argv++);
	}
	else if (strcasecmp (*argv, "reload") == 0)
	{
	    if (argc < 2)
	    {
		fprintf (stderr, "%s: need a argument for 'reload'\n", PrgName);
		exit (1);
	    }
	    argv++;
	    argc -= 2;
	    ReloadCacheTime = atoi (*argv++);
	}
	else if (strcasecmp (*argv, "log") == 0)
	{
	    if (argc < 2)
	    {
		fprintf (stderr, "%s: need a argument for 'log'\n", PrgName);
		exit (1);
	    }
	    argv++;
	    argc -= 2;
	    LogFile = *argv++;
	}
	else if (strcasecmp (*argv, "numreq") == 0)
	{
	    if (argc < 2)
	    {
		fprintf (stderr, "%s: need a argument for 'numreq'\n", PrgName);
		exit (1);
	    }
	    argv++;
	    argc -= 2;
	    if ( (MaxRequests = atoi (*argv++)) <= MIN_REQUESTS)
	    {
		fprintf (stderr, "%s: argument 'numreq' needs nummeric value > %d\n", PrgName, MIN_REQUESTS);
		exit (1);
	    }
	    if (MaxRequests * 3 >= FD_SETSIZE - 4)
	    {
		fprintf (stderr, "%s: argument 'numreq' maximum value of %d exceeded\n", PrgName, (FD_SETSIZE - 5) / 3);
		exit (1);
	    }
	}
	else if (strcasecmp (*argv, "unread") == 0)
	{
	    argv++;
	    argc--;
	    CacheUnreadRequests = TRUE;
	}
	else if (strcasecmp (*argv, "offline") == 0)
	{
	    argv++;
	    argc--;
	    if (GetQueued)
	    {
		fprintf (stderr, "%s: you cannot specify both, 'get' and 'offline'", PrgName);
		exit (1);
	    }
	    OffLine = 1;
	}
	else if (strcasecmp (*argv, "get") == 0)
	{
	    argv++;
	    argc--;
	    if (OffLine)
	    {
		fprintf (stderr, "%s: you cannot specify both, 'get' and 'offline'", PrgName);
		exit (1);
	    }
	    GetQueued = 1;
	}
	else
	{
	    fprintf (stderr, "%s: unknown option '%s'\n", PrgName, *argv);
	    exit (1);
	}
    }

    assert (FD_SETSIZE > MaxRequests * 3);

    Init (ProxyHost, ProxyPort, LogFile);

    if (ReadCacheList ())               /* Read cache entries or rebuild them */
	BuildCache ();

	/* Never return */
    for (;;)
    {
	CheckGetQueued ();
	BuildFdSets (&ReadSet, &WriteSet);
	debug (("Select... RequestsFree %d, CachesFree %d\n", RequestsFree, CachesFree));
	if (select (FD_SETSIZE, &ReadSet, &WriteSet, NULL, NULL) <= 0)
	{
	    if (errno == EINTR)
	    {
		fprintf (stderr, "%s: terminating due to signal - saving cachetable\n", PrgName);
		fprintf (LogStream, "terminating due to signal - saving cachetable\n");
		DeleteInvalidCaches ();
		SaveCacheList ();
		fprintf (stderr, "done\n", PrgName);
		fprintf (LogStream, "done\n\n");
		fclose  (LogStream);
		exit (0);
	    }
	    syslog (LOG_ERR, "%s: select failed: %s", PrgName, strerror (errno));
	}
	else
	{
	    ServRead (&ReadSet);
	    ServWrite (&WriteSet);
	}
    }
}
/*)) */