/*
    NewsCoaster
    Copyright (C) 1999-2002 Mark Harman

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

    NewsCoaster Homepage :
        http://newscoaster.tripod.com/

    Mailing List to hear about announcements of new releases etc at:
        http://groups.yahoo.com/group/newscoaster-announce/
*/

#include "mui_headers.h"
#include <mui/NListview_mcc.h>

#include <proto/socket.h>

#include <time.h>
#include <netdb.h>
#include <string.h>
#include <stdio.h>
#include <ctype.h>

#include "Vector.h"
#include "main.h"
#include "various.h"
#include "WriteWindow.h"
#include "Indices.h"
#include "misc.h"
#include "DateHandler.h"
#include "StatusWindow.h"
#include "Connection.h"
#include "netstuff.h"
#include "strings.h"
#include "ViewWindow.h"
#include "lists.h"

int onlineflag_len = strlen(onlineflag);

static char newline[3] = {13,10,0};

Connection *online_c = NULL;

void initNetstuff() {
	SocketBase=NULL;
}

/* Hang up the connection.
 */
void hangUp(Connection *c) {
	c->send_data("quit\r\n");
	c->close();
}

/* Authenticate to the newserver 'server' through connection 'c'.
 */
int auth(Connection *c,Server *server,StatusWindow * statusWindow) {
	nLog("auth((Connection *)%d,(Server *)%d,(StatusWindow *)statusWindow) called\n",c,server,statusWindow);
	char *line_buffer = new char[MAXLINE+1];
	//char buffer[4097] = "";
	if(server->nntp_auth) {
		statusWindow->setText("Authenticating..\n");
		sprintf(line_buffer,"AUTHINFO USER %s%s",server->user,newline);
		c->send_data(line_buffer);
		c->read_data(line_buffer);
		StripNewLine(line_buffer);
		if(*line_buffer!='3') {
			nLog("USER Error: %s\n",line_buffer);
			MUI_RequestA(app,0,0,"Error!","_Okay","\33Authentication Failed - Unknown User",0);
			delete [] line_buffer;
			return FALSE;
		}
		nLog("AAA\n");
		/*do_input();
		if(statusWindow->isAborted()==TRUE || running==FALSE) {
			hangUp(c);
			return FALSE;
		}*/

		sprintf(line_buffer,"AUTHINFO PASS %s%s",server->password,newline);
		c->send_data(line_buffer);
		c->read_data(line_buffer);
		StripNewLine(line_buffer);
		if(*line_buffer=='5') {
			nLog("PASS Error: %s\n",line_buffer);
			MUI_RequestA(app,0,0,"Error!","_Okay","\33Authentication Failed - Incorrect Password",0);
			delete [] line_buffer;
			return FALSE;
		}
		nLog("BBB\n");
		/*do_input();
		if(statusWindow->isAborted()==TRUE || running==FALSE) {
			hangUp(c);
			return FALSE;
		}*/
		statusWindow->setText("In!\n");
		nLog("CCC\n");
	}
	// no call to doOnlineBlocking() !
	delete [] line_buffer;
	return TRUE;
}

/* Downloads a newsgroup listing from 'server'.
 */
int getgrouplist(Server *server) {
	nLog("getgrouplist((Server *)%d) called\n",server);
	closeSockets();
	Connection *c = new Connection();
	if(!c->init()) {
		MUI_RequestA(app,0,0,"Error!","_Okay","\33cPlease start a\nTCP stack!",0);
		delete c;
		return -1;
	}
	// open a socket
	char *line_buffer = new char[MAXLINE+1];
	StatusWindow *statusWindow = new StatusWindow(app,"Downloading Group List");
	sprintf(status_buffer_g,"Connecting with '%s'...\n",server->nntp);
	statusWindow->setText(status_buffer_g);
	if(!c->call_socket(server)) {
		MUI_RequestA(app,0,0,"Error!","_Okay","\33cCannot connect with Newsserver!",0);
		delete c;
		delete [] line_buffer;
		delete statusWindow;
		return -1;
	}
	statusWindow->setText("Connected!\n");
	c->read_data(line_buffer);
	StripNewLine(line_buffer);
	if(*line_buffer!='2') {
		sprintf(status_buffer_g,"\33cCannot use this Newsserver!\nError: %s",line_buffer);
		MUI_RequestA(app,0,0,"Error!","_Okay",status_buffer_g,0);
		c->send_data("quit\r\n");
		delete c;
		delete [] line_buffer;
		delete statusWindow;
		return -1;
	}
	//authenticate
	if(!auth(c,server,statusWindow)) {
		//sprintf(status_buffer_g,"\33cCannot Log Into Newsserver!\nError: %s",line_buffer);
		//MUI_RequestA(app,0,0,"Error!","_Okay",status_buffer_g,0);
		sprintf(line_buffer,"quit%s",newline);
		c->send_data(line_buffer);
		delete c;
		delete [] line_buffer;
		delete statusWindow;
		return -1;
	}
	sprintf(line_buffer,"DATE%s",newline);
	c->send_data(line_buffer);
	c->read_data(line_buffer);
	StripNewLine(line_buffer);
	char lastdate[256] = "";
	BOOL gotdate = FALSE;
	if(strncmp(line_buffer,"111",3)!=0) {
		nLog("Warning! - Cannot get date from newsserver!\n%s\n''Get New Newsgroups' won't work!\n",line_buffer);
	}
	else {
		word(lastdate,line_buffer,2);
		if(*lastdate != '\0') {
			gotdate = TRUE;
		}
		else {
			nLog("Warning! - Cannot get date from newsserver!\n%s\n''Get New Newsgroups' won't work!\n",line_buffer);
		}
	}
	
	sprintf(line_buffer,"LIST%s",newline);
	c->send_data(line_buffer);
	c->read_data_line(line_buffer);
	StripNewLine(line_buffer);
	if(*line_buffer!='2') {
		sprintf(status_buffer_g,"\33cCannot Obtain GroupList!\nError: %s",line_buffer);
		MUI_RequestA(app,0,0,"Error!","_Okay",status_buffer_g,0);
		c->send_data("quit\r\n");
		delete c;
		delete [] line_buffer;
		delete statusWindow;
		return -1;
	}
	char filename[300] = "";
	sprintf(filename,"NewsCoasterData:%s.gl",server->nntp);
	DeleteFile(filename);
	BPTR file = Open(filename,MODE_NEWFILE);
	delete_dis = TRUE;
	int found = 0;
	if(file) {
		int len = 0;
		char *big_buffer = new char[big_bufsize_g+1];
		do {
			len = c->read_data_chunk(&big_buffer[8],big_bufsize_g-9);
			// count groups
			char *str = big_buffer+7;
			while(str = strchr(str+1,'\n'))
				found++;
			Write(file,&big_buffer[8],len);
			sprintf(status_buffer_g,"Found %d Groups...\n",found);
			statusWindow->setText(status_buffer_g);
			do_input();
			if(statusWindow->isAborted()==TRUE || running==FALSE)
				break;
			for(int l=3;l<8;l++)
				big_buffer[l] = big_buffer[big_bufsize_g-9+l];
		} while(len == big_bufsize_g-9);
		delete [] big_buffer;
		Close(file);
		file = NULL;

		if(gotdate) {
			strncpy(server->lastGotGroups,lastdate,14);
			server->lastGotGroups[14] = '\0';
			account.save_data();
			nLog("DATE: *%s*\n",server->lastGotGroups);
		}
	}
	else {
			nLog("Warning! - Cannot open file %s!\n",filename);
	}

	delete_dis=FALSE;
	c->send_data("quit\r\n");
	delete c;
	delete [] line_buffer;
	delete statusWindow;
	return found;
}

/* Download new newsgroups from 'server'.
 */
BOOL getnewgroups(Server *server) {
	nLog("getnewgroups((Server *)%d) called\n",server);
	char date[15]="";
	strncpy(date,server->lastGotGroups,14);
	date[14] = '\0';
	nLog("Date *%s*\n",date);
	BOOL okay = TRUE;
	if(strlen(date)!=14)
		okay=FALSE;
	else {
		for(int k=0;k<14;k++) {
			if(isdigit(date[k])==FALSE) {
				okay=FALSE;
				break;
			}
		}
	}
	if(!okay) {
		sprintf(status_buffer_g,"\33cDate of last newsgroupslist download not available!\nPlease try 'Fetch Group List' first.\n");
		MUI_RequestA(app,0,0,"Error!","_Okay",status_buffer_g,0);
		return FALSE;
	}
	closeSockets();
	Connection *c = new Connection();
	if(!c->init()) {
		MUI_RequestA(app,0,0,"Error!","_Okay","\33cPlease start a\nTCP stack!",0);
		delete c;
		return -1;
	}
	
	StatusWindow *statusWindow = new StatusWindow(app,"Downloading New Newsgroups..");
	sprintf(status_buffer_g,"Connecting with '%s'...\n",server->nntp);
	statusWindow->setText(status_buffer_g);
	if(!c->call_socket(server)) {
		MUI_RequestA(app,0,0,"Error!","_Okay","\33cCannot connect with Newsserver!",0);
		delete c;
		delete statusWindow;
		return FALSE;
	}
	char *line_buffer = new char[MAXLINE + 1];
	statusWindow->setText("Connected!\n");
	c->read_data(line_buffer);
	StripNewLine(line_buffer);
	if(*line_buffer!='2') {
		sprintf(status_buffer_g,"\33cCannot use this Newsserver!\nError: %s",line_buffer);
		MUI_RequestA(app,0,0,"Error!","_Okay",status_buffer_g,0);
		c->send_data("quit\r\n");
		delete c;
		delete [] line_buffer;
		delete statusWindow;
		return FALSE;
	}
	//authenticate
	if(!auth(c,server,statusWindow)) {
		//sprintf(status_buffer_g,"\33cCannot Log Into Newsserver!\nError: %s",line_buffer);
		//MUI_RequestA(app,0,0,"Error!","_Okay",status_buffer_g,0);
		c->send_data("quit\r\n");
		delete c;
		delete [] line_buffer;
		delete statusWindow;
		return FALSE;
	}
	char newdate[16]="";
	strncpy(newdate,&date[2],6);
	newdate[6]=' ';
	strncpy(&newdate[7],&date[8],6);
	newdate[13]=0;
	
	sprintf(line_buffer,"DATE%s",newline);
	c->send_data(line_buffer);
	c->read_data(line_buffer);
	StripNewLine(line_buffer);
	char lastdate[256] = "";
	BOOL gotdate = FALSE;
	if(strncmp(line_buffer,"111",3)!=0) {
		nLog("Warning! - Cannot get date from newsserver!\n%s\n''Get New Newsgroups' won't work!\n",line_buffer);
	}
	else {
		word(lastdate,line_buffer,2);
		if(*lastdate != '\0') {
			gotdate = TRUE;
		}
		else {
			nLog("Warning! - Cannot get date from newsserver!\n%s\n''Get New Newsgroups' won't work!\n",line_buffer);
		}
	}
	
	sprintf(line_buffer,"NEWGROUPS %s%s",newdate,newline);
	c->send_data(line_buffer);
	c->read_data_line(line_buffer);
	StripNewLine(line_buffer);
	if(*line_buffer!='2') {
		sprintf(status_buffer_g,"\33cCannot Obtain New NewsGroups List!\nError: %s",line_buffer);
		MUI_RequestA(app,0,0,"Error!","_Okay",status_buffer_g,0);
		c->send_data("quit\r\n");
		delete c;
		delete [] line_buffer;
		delete statusWindow;
		return FALSE;
	}
	char filename[300] = "";
	sprintf(filename,"NewsCoasterData:%s.gl",server->nntp);
	BPTR lock=Lock(filename,ACCESS_READ);
	BPTR file=Open(filename,MODE_OLDFILE);
	delete_dis=TRUE;
	if(file) {
		int found=0;
		Seek(file,0,OFFSET_END);
		int filepos = Seek(file,0,OFFSET_CURRENT);
		Seek(file,0,OFFSET_END);
		char *big_buffer = new char[big_bufsize_g + 1];
		strncpy(&big_buffer[6],newline,2);
		int len = 0;
		do {
			//printf("!\n");
			DoMethod(app,MUIM_Application_InputBuffered);
			len = c->read_data_chunk(&big_buffer[8],big_bufsize_g-9);
			//printf("%d : *%s*\n",len,&big_buffer[8]);
			// count groups
			char *str = &big_buffer[7];
			while(str = strchr(str+1,'\n'))
				found++;
			Write(file,&big_buffer[8],len);
			sprintf(status_buffer_g,"Found %d New Groups...\n",found);
			statusWindow->setText(status_buffer_g);
			do_input();
			if(statusWindow->isAborted()==TRUE || running==FALSE)
				break;
			for(int l=3;l<8;l++)
				big_buffer[l] = big_buffer[big_bufsize_g-9+l];
		} while(len==big_bufsize_g-9);
		if(found==0) {
			sprintf(status_buffer_g,"\33cFound %d New Groups!\n",found);
			MUI_RequestA(app,0,0,"Get New Groups","_Okay",status_buffer_g,0);
		}
		Close(file);
		file = NULL;
		
		if(gotdate) {
			strncpy(server->lastGotGroups,lastdate,14);
			server->lastGotGroups[14] = '\0';
			account.save_data();
			nLog("DATE: *%s*\n",server->lastGotGroups);
		}
		
		if(found>0) {
			groupman_server = server;
			//readgrouplist(server,NLIST_groupman,filepos);
			readgrouplist(server,filepos);
			set(wnd_groupman,MUIA_Window_Open,TRUE);
		}
	}
	else {
			nLog("Warning! - Cannot open file %s!\n",filename);
	}
	delete_dis=FALSE;
	if(lock) {
		UnLock(lock);
		lock=NULL;
	}
	c->send_data("quit\r\n");
	delete c;
	delete [] line_buffer;
	delete statusWindow;
	return TRUE;
}

/* Log into the newsserver.
 */
BOOL logIn(Connection *c,Server * server,StatusWindow * sw) {
	nLog("logIn((Connection *)%d,(Server *)%d,(StatusWindow *)%d) called\n",c,server,sw);
	//char buffer[MAXLINE+1]="";
	//char temp[1024]="";
	if(!c->call_socket(server)) {
		MUI_RequestA(app,0,0,"Error!","_Okay","\33cCannot connect with Newsserver!",0);
		return FALSE;
	}
	do_input();
	if(sw->isAborted()==TRUE || running==FALSE) {
		c->close(); // exit immediately
		return FALSE;
	}

	char *line_buffer = new char[MAXLINE+1];
	c->read_data(line_buffer);
	StripNewLine(line_buffer);
	if(*line_buffer!='2') {
		sprintf(status_buffer_g,"\33cCannot use this Newsserver\nError: %s",line_buffer);
		MUI_RequestA(app,0,0,"Error!","_Okay",status_buffer_g,0);
		hangUp(c);
		delete [] line_buffer;
		return FALSE;
	}
	do_input();
	if(sw->isAborted()==TRUE || running==FALSE) {
		c->close(); // exit immediately
		delete [] line_buffer;
		return FALSE;
	}

	//authenticate
	if(!auth(c,server,sw)) {
		//sprintf(status_buffer_g,"\33cCannot Log Into Newsserver!\nError: %s",line_buffer);
		//MUI_RequestA(app,0,0,"Error!","_Okay",status_buffer_g,0);
		hangUp(c);
		delete [] line_buffer;
		return FALSE;
	}

	//mode reader
	sprintf(line_buffer,"mode reader%s",newline);
	c->send_data(line_buffer);
	c->read_data(line_buffer);
	StripNewLine(line_buffer);
	if(*line_buffer!='2' && *line_buffer!='5') {
		nLog("Warning: %s received from 'mode reader' command\n",line_buffer);
	}
	do_input();
	if(sw->isAborted()==TRUE || running==FALSE) {
		hangUp(c);
		delete [] line_buffer;
		return FALSE;
	}

	delete [] line_buffer;
	return TRUE;
}

/* Post outgoing messages.
 * TODO: this doesn't really cope well with messages that are to be both
 * posted and emailed, if one of those fails; at the moment in such cases,
 * the message remains in outgoing, and will be both posted/emailed next
 * time (ie, even though one of those has already been done).
 */
void postnews(int type) {
	// type: 0=normal, 1=post online (immediate) only (flags[2]==TRUE)
	nLog("postnews((int)%d) called\n",type);
	closeSockets();
	Connection *c = new Connection();
	if(!c->init()) {
		MUI_RequestA(app,0,0,"Error!","_Okay","\33cPlease start a\nTCP stack!",0);
		delete c;
		return;
	}
	char *line_buffer = new char[MAXLINE+1];
	Server *current_server = NULL;
	BPTR fileout=NULL;
	GroupData * gdata=NULL;
	GroupData * gdata_sent=NULL;
	get_gdata(&gdata,".outgoing");
	get_gdata(&gdata_sent,".sent");
	set_and_read(gdata);
	setGdataQuiet(TRUE);
	setMdataQuiet(TRUE);
	BOOL newsup=FALSE,emailup=FALSE;
	//pre-scan
	int entries = 0;
	get(NLIST_messagelistdata,MUIA_NList_Entries,&entries);
	BOOL * news_sent = new BOOL[entries];
	BOOL * email_sent = new BOOL[entries];
	for(int k=0;k<entries;k++) {
		MessageListData * mdata = NULL;
		DoMethod(NLIST_messagelistdata,MUIM_NList_GetEntry,k,&mdata);
		mdata->flags[3] = mdata->flags[2];
		mdata->flags[2] = FALSE; // need to unset post-immediately flag in case we fail too early!
		if( (mdata->flags[0]==FALSE && type==0) || (mdata->flags[3]==TRUE && type==1) ) {
			if(mdata->flags[4]==TRUE)
				newsup=TRUE;
			if(mdata->flags[5]==TRUE)
				emailup=TRUE;
		}
		news_sent[k]=FALSE;
		email_sent[k]=FALSE;
	}
	char filename[300] = "";
	if(newsup) {
		//BOOL cont = TRUE;
		// open a socket to newsserver
		StatusWindow *statusWindow = new StatusWindow(app,"Sending News..");
		//post messages
		delete_dis=TRUE;
		//if(cont)
		{
			for(int k=0;k<entries;k++) {
				MessageListData * mdata = NULL;
				DoMethod(NLIST_messagelistdata,MUIM_NList_GetEntry,k,&mdata);
				BOOL post=FALSE;
				if(type==0 && mdata->flags[0]==FALSE)
					post=TRUE;
				else if(type==1 && mdata->flags[0]==FALSE && mdata->flags[3]==TRUE)
					post=TRUE;
				if(mdata->flags[4]==FALSE)
					post=FALSE;
				if(post) {
					// try to fathom out which newsserver to use
					//printf("***%s\n",mdata->newsgroups);
					char ngroups[1024] = "";
					char *delim = ",";
					//sprintf(ngroups,"%s ",mdata->newsgroups);
					get_newsgroups(ngroups,gdata->ID,mdata->ID);
					char *grp = strtok(ngroups,delim);
					/*while(grp != NULL) {
						printf("group: %s\n",grp);
						Server *server = getPostingServerForGroup(grp);
						printf("server: %s\n",server->nntp);
						grp = strtok(NULL,delim);
					}*/
					//printf("group: %s\n",grp);
					Server *this_server = getPostingServerForGroup(grp);
					//printf("server: %s\n",this_server->nntp);
					
					if(this_server != current_server) {
						// change servers
						//printf("changing servers..\n");
						if(current_server != NULL) {
							// disconnect
							if(c->connected())
								hangUp(c);
						}
						// reconnect
						current_server = this_server;
						sprintf(status_buffer_g,"Connecting with '%s'...\n",current_server->nntp);
						statusWindow->setText(status_buffer_g);

						if(!logIn(c,current_server,statusWindow)) {
							current_server = NULL;
							continue;
						}
					}
					
					sprintf(line_buffer,"Sending: %s\n",mdata->subject);
					statusWindow->setText(line_buffer);
					
					//continue;
					
					sprintf(filename,"NewsCoasterData:outgoing/news_%d",mdata->ID);
					fileout=Open(filename,MODE_OLDFILE);
					if(fileout) {
						sprintf(line_buffer,"POST%s",newline);
						c->send_data(line_buffer);
						c->read_data(line_buffer);
						StripNewLine(line_buffer);
						if(*line_buffer!='3' && *line_buffer!='2' && *line_buffer!='4') {
							MUI_RequestA(app,0,0,"Error!","_Okay","\33cCannot post to this Newsserver!",0);
							Close(fileout);
							fileout=NULL;
							//cont=FALSE;
							break;
						}
						else {
							char *eof=FGets(fileout,line_buffer,MAXLINE);
							while(eof) {
								StripNewLine(line_buffer);
								do_input(); // input loop
								if(statusWindow->isAborted()==TRUE || running==FALSE)
									break;
								sprintf(line_buffer,"%s%s",line_buffer,newline);
								if(*line_buffer=='.')
									c->send_data(".");
								c->send_data(line_buffer);
								eof=FGets(fileout,line_buffer,MAXLINE);
							}
							Close(fileout);
							fileout=NULL;
							if(statusWindow->isAborted()==FALSE && running==TRUE) {
								sprintf(line_buffer,"%s.%s",newline,newline);
								c->send_data(line_buffer);
								c->read_data(line_buffer);
								StripNewLine(line_buffer);
								if(*line_buffer!='2') {
									sprintf(status_buffer_g,"Error: %s",line_buffer);
									MUI_RequestA(app,0,0,"Error - Post Failed!","_Okay",status_buffer_g,0);
								}
								else {
									statusWindow->setText("Done\n");
									news_sent[k]=TRUE;
								}
							}
							else
								statusWindow->setText("Aborted!\n");
						}
					}
					else {
						printf("Error! Cannot open file: %s\n",filename);
						sprintf(status_buffer_g,"Error! Cannot open file:\n  %s",filename);
						statusWindow->setText(status_buffer_g);
					}
				}
				if(statusWindow->isAborted()==TRUE || running==FALSE)
					break;
			}
		}
		if(c->connected())
			hangUp(c);
		delete statusWindow;
	}
	
	if(emailup) {
		BOOL cont = TRUE;
		// open a socket to smtp server
		StatusWindow *statusWindow = new StatusWindow(app,"Sending Email..");
		sprintf(status_buffer_g,"Connecting with '%s'...\n",account.smtp);
		statusWindow->setText(status_buffer_g);
		if(c->connected()) {
			c->send_data("quit\r\n");
			c->close();
		}
		if(!c->call_socket(account.smtp,25)) {
			MUI_RequestA(app,0,0,"Error!","_Okay","\33cCannot connect with SMTP server!",0);
			cont=FALSE;
		}
		if(cont) {
			statusWindow->setText("Connected!\n");
			c->read_data(line_buffer);
			nLog("  received: %s\n",line_buffer);
			StripNewLine(line_buffer);
			if(*line_buffer!='2') {
				sprintf(status_buffer_g,"\33cCannot use this SMTP server!\nError: %s",line_buffer);
				MUI_RequestA(app,0,0,"Error!","_Okay",status_buffer_g,0);
				cont=FALSE;
			}
		}
		if(cont) {
			char *remail = ( *account.realemail != '\0' ) ? account.realemail : account.email;
			char *ptr = strchr(remail,'@');
			if(ptr != NULL)
				remail = &ptr[1];
			sprintf(line_buffer,"HELO %s%s",remail,newline);
			nLog("  sending: %s",line_buffer);
			c->send_data(line_buffer);
			c->read_data(line_buffer);
			nLog("  received: %s\n",line_buffer);
			StripNewLine(line_buffer);
			if(*line_buffer != '2') {
				sprintf(line_buffer,"HELO %s",remail);
				char *ptr = strchr(line_buffer,'@');
				if(ptr != NULL) {
					// try again
					strcpy(ptr,newline);
					nLog("  sending: %s",line_buffer);
					c->send_data(line_buffer);
					c->read_data(line_buffer);
					nLog("  received: %s\n",line_buffer);
					StripNewLine(line_buffer);
				}
				if(*line_buffer != '2') {
					// try yet again
					sprintf(line_buffer,"HELO%s",newline);
					nLog("  sending: %s",line_buffer);
					c->send_data(line_buffer);
					c->read_data(line_buffer);
					nLog("  received: %s\n",line_buffer);
					StripNewLine(line_buffer);
				}
				if(*line_buffer != '2') {
					// failed
					sprintf(status_buffer_g,"\33cCannot send to this SMTP server!\nError: %s",line_buffer);
					MUI_RequestA(app,0,0,"Error!","_Okay",status_buffer_g,0);
					cont=FALSE;
				}
			}
			for(int k=0;cont && k<entries;k++) {
				MessageListData * mdata = NULL;
				DoMethod(NLIST_messagelistdata,MUIM_NList_GetEntry,k,&mdata);
				BOOL post = FALSE;
				if(type==0 && mdata->flags[0]==FALSE)
					post = TRUE;
				else if(type==1 && mdata->flags[0]==FALSE && mdata->flags[3]==TRUE)
					post = TRUE;
				if(mdata->flags[5]==FALSE)
					post = FALSE;
				if(post) {
					sprintf(line_buffer,"Sending: %s\n",mdata->subject);
					statusWindow->setText(line_buffer);
					sprintf(filename,"NewsCoasterData:outgoing/news_%d",mdata->ID);
					fileout=Open(filename,MODE_OLDFILE);
					if(fileout) {
						NewMessage newmessage;
						strcpy(newmessage.getThisHeader,"to:");
						get_refs(&newmessage,gdata,mdata,GETREFS_NONE);
						if(*account.realemail!=0)
							sprintf(line_buffer,"MAIL FROM: <%s>%s",account.realemail,newline);
						else
							sprintf(line_buffer,"MAIL FROM: <%s>%s",account.email,newline);
						nLog("  sending: %s",line_buffer);
						c->send_data(line_buffer);
						c->read_data(line_buffer);
						nLog("  received: %s\n",line_buffer);
						StripNewLine(line_buffer);
						if(*line_buffer!='2') {
							sprintf(status_buffer_g,"\33cCannot send to this SMTP server!\nError: %s",line_buffer);
							MUI_RequestA(app,0,0,"Error!","_Okay",status_buffer_g,0);
							Close(fileout);
							fileout=NULL;
							cont=FALSE;
							break;
						}
						char buffer2[256] = "";
						int rcpt_no=1;
						while(cont) {
							char rcpt[256] = "";
							//word(rcpt,newmessage.to,rcpt_no,',');
							word(rcpt,newmessage.dummyHeader,rcpt_no,',');
							if(*rcpt == NULL)
								break;
							get_email(buffer2,rcpt,GETEMAIL_EMAIL);
							sprintf(line_buffer,"RCPT TO: <%s>%s",buffer2,newline);
							nLog("  sending: %s",line_buffer);
							c->send_data(line_buffer);
							c->read_data(line_buffer);
							nLog("  received: %s\n",line_buffer);
							StripNewLine(line_buffer);
							if(*line_buffer!='2') {
								sprintf(status_buffer_g,"\33cError for recipient %s:\n%s",rcpt,line_buffer);
								MUI_RequestA(app,0,0,"Error!","_Okay",status_buffer_g,0);
								Close(fileout);
								fileout=NULL;
								cont=FALSE;
								break;
							}
							rcpt_no++;
						}
						if(cont) {
							sprintf(line_buffer,"DATA%s",newline);
							c->send_data(line_buffer);
							c->read_data(line_buffer);
							StripNewLine(line_buffer);
							char *eof=FGets(fileout,line_buffer,MAXLINE);
							while(eof) {
								StripNewLine(line_buffer);
								do_input(); // input loop
								if(statusWindow->isAborted()==TRUE || running==FALSE)
									break;
								sprintf(line_buffer,"%s%s",line_buffer,newline);
								if(*line_buffer=='.')
									c->send_data(".");
								c->send_data(line_buffer);
								eof=FGets(fileout,line_buffer,MAXLINE);
							}
							Close(fileout);
							fileout=NULL;
							if(statusWindow->isAborted()==FALSE && running==TRUE) {
								sprintf(line_buffer,"%s.%s",newline,newline);
								c->send_data(line_buffer);
								c->read_data(line_buffer);
								StripNewLine(line_buffer);
								if(*line_buffer!='2') {
									nLog("Email Failed: %s\n",line_buffer);
									sprintf(status_buffer_g,"Email Failed:\n%s",line_buffer);
									statusWindow->setText(status_buffer_g);
								}
								else {
									statusWindow->setText("Done\n");
									email_sent[k]=TRUE;
								}
							}
							else {
								statusWindow->setText("Aborted!\n");
							}
						}
					}
					else {
						sprintf(status_buffer_g,"Error! Cannot open file:\n  %s",filename);
						statusWindow->setText(status_buffer_g);
					}
				}
				if(statusWindow->isAborted()==TRUE || running==FALSE)
					break;
			}
		}
		c->send_data("quit\r\n");
		c->close();
		delete statusWindow;
	}
	
	// copy sent messages to .sent
	if(entries>0) {
		MessageListData ** mdata_sent = new MessageListData *[entries];
		int sent=0;
		for(int k=0;k<entries;k++) {
			MessageListData * mdata = NULL;
			DoMethod(NLIST_messagelistdata,MUIM_NList_GetEntry,k,&mdata);
			if(mdata->flags[4]==TRUE || mdata->flags[5]==TRUE) {
				// only move to .sent if it has both posted/emailed as required
				if( (news_sent[k]==TRUE || mdata->flags[4]==FALSE) && (email_sent[k]==TRUE || mdata->flags[5]==FALSE) )
					mdata_sent[sent++]=mdata;
				// if one of emailing/posting was successful, turn off so as to not resend
				if(news_sent[k]==TRUE && mdata->flags[4]==TRUE)
					mdata->flags[4]=FALSE;
				if(email_sent[k]==TRUE && mdata->flags[5]==TRUE)
					mdata->flags[5]=FALSE;
			}
		}
		if(sent>0) {
			StatusWindow *statusWindow = new StatusWindow(app,"Moving to sent..");
			set_and_read(gdata);
			set(wnd_main,MUIA_Window_Sleep,TRUE);
			WriteWindow::sleepAll(TRUE);
			for(int k=0;k<sent;k++) {
				sprintf(status_buffer_g,"Moving %d to sent..\n",k+1);
				statusWindow->setText(status_buffer_g);
				move(gdata,gdata_sent,mdata_sent[k],0,FALSE,FALSE);
			}
			write_index(0);
			read_index();
			set(wnd_main,MUIA_Window_Sleep,FALSE);
			WriteWindow::sleepAll(FALSE);
			delete statusWindow;
		}
		if(mdata_sent) {
			delete [] mdata_sent;
			mdata_sent=NULL;
		}
	}
	delete_dis=FALSE;
	setGdataQuiet(FALSE);
	setMdataQuiet(FALSE);
	if(news_sent) {
		delete [] news_sent;
		news_sent=NULL;
	}
	if(email_sent) {
		delete [] email_sent;
		email_sent=NULL;
	}
	delete c;
	delete [] line_buffer;
	return;
}

/* Being lazy and using some globals.. */
int        histlen                = 0;
char     **mIDhist                = NULL;
int        nkills                 = 0;
KillFile **killfiles              = NULL;
BOOL      *replace_killfile       = NULL;
BOOL      *killfile_in_this_group = NULL;
int       *killfile_types         = NULL;
BOOL       on_refs                = FALSE;
char       isobuf[4096]           = "";

int killArticle(int k,int l,BOOL *important,StatusWindow *statusWindow) {
	BOOL skip = FALSE;
	//char status[256] = "";
	KillFile *kill = killfiles[l];

	*important = FALSE;

											nLog("*** Message Killfile match! - %s %s %s (%d, %d)\n",kill->header,kill->text,kill->ngroups,k,l);
											nLog("    match = %d ; action = %d\n",kill->match,kill->action);
											if(kill->action==0) {
												skip = TRUE;
												sprintf(status_buffer_g,"Message %d Killed! (%s match)\n",k,kill->header);
												statusWindow->setText(status_buffer_g);
											}
											else if(kill->action==1)
												*important = TRUE;

											if(!replace_killfile[l]) {
												KillFile * newkill = new KillFile();
												strncpy(newkill->header,kill->header,63);
												strncpy(newkill->text,kill->text,255);
												strncpy(newkill->ngroups,kill->ngroups,63);
												newkill->header[63] = '\0';
												newkill->text[255] = '\0';
												newkill->ngroups[63] = '\0';
												newkill->ds = kill->ds;
												// lastused is at current datetime now - see KillFile constructor
												newkill->expiretype = kill->expiretype;
												newkill->expire = kill->expire;
												newkill->match = kill->match;
												newkill->action = kill->action;
												newkill->carryon = kill->carryon;
												nLog("  about to replace killfile..\n");
												// copy it back
												*kill = *newkill;
												replace_killfile[l] = TRUE;
												delete newkill;
											}
											nLog("  done\n");
	return skip;
}

/* Process the line of header in 'buffer'.
 */
int processHeader(char * buffer,GroupData * gdata,MessageListData * mdata,char * ngroup,StatusWindow *statusWindow,int article) {
	nLog("processHeader((char *)%s,...,(MessageListData *)%d,...,(int)%d) called\n",buffer,mdata,article);
	int skip = FALSE;

	const int MAXWORD = 512;
	char *ptr = strstr(buffer,": ");
	if( ptr == NULL || (int)(ptr-buffer) > MAXWORD )
		return FALSE;

	char word1[MAXWORD + 1] = "";
	//char status[256] = "";
	int w1len = wordFirstAndLenUpper(word1,buffer);
	//killfile
	for(int l=0;l<nkills;l++) {
		KillFile *kill = killfiles[l];

		if(*kill->header=='\0' || equals(word1,kill->header)) {
			//matched header
			char *match = stristr_q(&buffer[w1len+1],kill->text);
			//if(stristr_q(&buffer[w1len+1],kill->text)) {
			if( (kill->match==0 && match!= NULL) || (kill->match==1 && match==NULL) ) {
				//matched text
				//if(stristr_q(ngroup,kill->ngroups)!=NULL) {
				if(killfile_in_this_group[l]) {
					//matched group
					// kill it!
					nLog("*** Message Killfile match! - %s %s %s (%d, %d)\n",kill->header,kill->text,kill->ngroups,article,l);
					nLog("    match = %d ; action = %d\n",kill->match,kill->action);
					if(kill->action==0) {
						skip = TRUE;
						sprintf(status_buffer_g,"Message %d Killed! (%s match)\n",article,kill->header);
						statusWindow->setText(status_buffer_g);
					}
					else if(kill->action==1)
						mdata->flags[13] = 1;
					if(!replace_killfile[l]) {
						KillFile * newkill = new KillFile();
						strncpy(newkill->header,kill->header,63);
						strncpy(newkill->text,kill->text,255);
						strncpy(newkill->ngroups,kill->ngroups,63);
						newkill->header[63] = '\0';
						newkill->text[255] = '\0';
						newkill->ngroups[63] = '\0';
						newkill->ds = kill->ds;
						// lastused is at current datetime now - see KillFile constructor
						newkill->expiretype = kill->expiretype;
						newkill->expire = kill->expire;
						//newkill->type = kill->type;
						newkill->match = kill->match;
						newkill->action = kill->action;
						newkill->carryon = kill->carryon;
						nLog("  about to replace killfile..\n");
						// copy it back
						*kill = *newkill;
						replace_killfile[l] = TRUE;
						delete newkill;
					}
					nLog("  done\n");
					if(!kill->carryon) {
						nLog("skip rest\n");
						break;
					}
				}
			}
		}
	}
	//nLog("finished kills\n");
	
	
	if(!skip) {
		// cope with multiple line references
		if(on_refs) {
			//nLog("on_refs\n");
			if(strchr(word1,':') != NULL)
				on_refs = FALSE;
			else {
				char *lastref = NULL;
				char *ptr = buffer;
				for(;;) {
					ptr = strchr(ptr,'<');
					if(ptr == NULL)
						break;
					lastref = ptr;
					ptr++;
				}
				if(lastref != NULL) {
					//nLog("lastref: %s\n",lastref);
					int i=0;
					while(*lastref != NULL) {
						mdata->lastref[i] = *lastref;
						i++;
						if(*lastref=='>' || i==IHEADENDMID)
							break;
						lastref++;
					}
					mdata->lastref[i] = '\0';
				}
			}
		}
		//nLog("about to scan : %s\n",word1);

		if(strlen(word1) == strlen(buffer)) {
			// empty header
		}
		else if(strcmp(word1,"FROM:")==0)
		{
			//strncpy(mdata->from,&buffer[6],IHEADENDSHORT);
			translateIso(isobuf,&buffer[6]);
			strncpy(mdata->from,isobuf,IHEADENDSHORT);
			mdata->from[IHEADENDSHORT]=0;
			//stripEscapes(mdata->from);
		}
		else if(strcmp(word1,"SUBJECT:")==0)
		{
			//strncpy(mdata->subject,&buffer[9],IHEADENDSUBJECT);
			translateIso(isobuf,&buffer[9]);
			strncpy(mdata->subject,isobuf,IHEADENDSUBJECT);
			mdata->subject[IHEADENDSUBJECT]= '\0';
			//stripEscapes(mdata->subject);
			if(account.use_sizekill) {
				char *word2=wordLast(mdata->subject);
				if(word2) {
					if(*word2!=0) {
						BOOL okay=TRUE;
						int w2len = strlen(word2);
						for(int l=0;l<w2len;l++) {
							if(!isdigit((int)word2[l])) {
								okay=FALSE;
								break;
							}
						}
						if(okay) {
							if(strlen(word2)>=3) { // if number is 3 digits or more
								nLog("*** Message Killed! - 'Advert' %s (%d)\n",word2,article);
								sprintf(status_buffer_g,"Message %d Killed! ('Advert')\n",article);
								statusWindow->setText(status_buffer_g);
								skip=TRUE;
							}
						}
						word2=NULL;
					}
				}
			}
		}
		else if(strcmp(word1,"DATE:")==0)
		{
			*mdata->datec = '\0';
			//strncpy(mdata->date,&buffer[6],IHEADENDSHORT);
			//mdata->date[IHEADENDSHORT]=0;
			//DateHandler::read_date(&mdata->ds,mdata->date,mdata->c_date,mdata->c_time);
			DateHandler::read_date(&mdata->ds,&buffer[6],mdata->c_date,mdata->c_time);
		}
		else if(strcmp(word1,"MESSAGE-ID:")==0)
		{
			strncpy(mdata->messageID,&buffer[12],IHEADENDMID);
			mdata->messageID[IHEADENDMID]=0;
			//stripEscapes(mdata->messageID);
			char **hist = mIDhist;
			for(int l=0;l<histlen;l++) {
				if(strcmp(*hist++,mdata->messageID)==0) {
					// already downloaded!
					nLog("*** Duplicate Message! - %s (%d, %d)\n",mdata->messageID,article,l);
					sprintf(status_buffer_g,"Message %d Killed! (Duplicate Message)\n",article);
					statusWindow->setText(status_buffer_g);
					skip=TRUE;
					break;
				}
			}
		}
		else if(strcmp(word1,"REFERENCES:")==0)
		{
			char *lastref = wordLast(buffer);
			strncpy(mdata->lastref,lastref,IHEADENDMID);
			mdata->lastref[IHEADENDMID]=0;
			on_refs = TRUE;
		}
		else if(strcmp(word1,"LINES:")==0)
		{
			//char word2[256] = "";
			//wordFirst(word2,&buffer[w1len+1]);
			//if(*word2 != '\0')
			{
				//int tline = atoi(word2);
				//int tline = atoi(&buffer[w1len+1]);
				int tline = atoi(&buffer[7]);
				mdata->flags[8] = tline;
				// 'lines' filter
				if(gdata->flags[2]>0) {
					if(tline>gdata->flags[2]) {
						nLog("*** Message Killed! - %d longer than %d lines (%d)\n",tline,gdata->flags[2],article);
						sprintf(status_buffer_g,"Message %d Killed! (Too Many Lines)\n",article);
						statusWindow->setText(status_buffer_g);
						skip=TRUE;
					}
				}
			}
		}
	}
	// no more checks; they must go before LINES
	// no call to doOnlineBlocking() !
	//nLog("  returning %d\n",skip);
	return skip;
}

/* Downloads the a message; either the entire message, or just the body.
 */
BOOL download_body(BOOL *available,Connection *c,BPTR file,int *bread,int *bout,StatusWindow *statusWindow,BOOL entire_message,char *buffer) {
	nLog("download_body() called\n");
	BOOL in_header = entire_message;
	//char buffer[MAXLINE+1] = "";
	//char *buffer = new char[MAXLINE+1];
	char *bufptr = NULL;
	BOOL done = FALSE;
	BOOL res = TRUE;
	int br = 0;
	int bm = 0;
	int len = c->read_data(buffer,MAXLINE);
	if(len > 0) {
		br += len;
		if(*buffer!='2') {
			*available = FALSE;
			nLog("not available!\n");
			res = FALSE;
		}
		else {
   	   bufptr = strstr(buffer,"\r\n") + 2;
	   	bm += (int)bufptr - (int)buffer;
	   }
		if(res) {
			int l = 0;
			char line[MAXLINE+1] = "";
			//char *line = new char[MAXLINE+1];
			for(;;) {
				do_input();
				if(statusWindow->isAborted()==TRUE || running==FALSE) {
					res = FALSE;
					break;
				}
				while( bufptr < &buffer[len] ) {
					if (*bufptr != '\r')
						line[l++] = *bufptr;
					else
						bm++;
					if(l==MAXLINE-1) {
						line[l] = '\0';
						l = 0;
						if(FPuts(file,line) == -1) {
							res = FALSE;
							break;
						}
					}
					if(*bufptr++ != '\n')
						continue;
					line[l] = '\0';
					l = 0;
					if(line[0] == '.') {
						if(line[1] == '\n') {
							bm += 2;
							done = TRUE;
							break;
						}
						else {
							l = 1;
							bm++;
						}
					}
					if(in_header) {
						if(line[0] == '\n' && line[1] == '\0')
							in_header=  FALSE;
						else {
							stripEscapes(line);
						}
					}
					if(FPuts(file,&line[l]) == -1) {
						res = FALSE;
						break;
					}
					l = 0;
				}
				if(done)
					break;
				len = c->read_data(buffer,MAXLINE);
				if(len<=0)
					break;
				br += len;
				bufptr = buffer;
			}
			//delete [] line;
		}
	}
	else {
		nLog("no response from server!\n");
		res = FALSE;
	}
	//delete [] buffer;
	Flush(file);
	*bread = br;
	*bout = br - bm;
	nLog("returning %d\n",res);
	return res;
}

void sortGroupList(GroupData **gdata_list,int entries) {
	for(int dg=0;dg<entries-2;dg++) {
		Server *this_server = getServer( gdata_list[dg]->serverID );
		for(int j=dg+1;j<entries;j++) {
			Server *server = getServer( gdata_list[j]->serverID );
			if(server == this_server) {
				// swap with position dg+1
				if(j != dg+1) {
					GroupData *temp_gdata = gdata_list[j];
					gdata_list[j] = gdata_list[dg+1];
					gdata_list[dg+1] = temp_gdata;
				}
				break;
			}
		}
	}
}

/*
void killWithXHDR(BOOL *skip_articles,BOOL *mark_important,Connection *c,int start,int last,StatusWindow *statusWindow) {
	int nidz = last - start + 1;
	char status[256] = "";
	const int bufsize = 8192; // must be >= 1024 !!!
	char buffer[bufsize+1] = "";
	const int hdsize = 32768;
	char hdbuffer[hdsize+1] = "";
					BOOL *checked_articles = new BOOL[nidz];
					for(int i=0;i<nidz;i++)
						checked_articles[i] = FALSE;

					for(int l=0;l<nkills;l++) {
						if(killfile_in_this_group[l]) {
							int type = killfile_types[l];
							if(type != 0)
								continue;

							KillFile *kill = killfiles[l];
							char header[256] = "";
							strcpy(header,kill->header);
							StripChar(header,':');
							sprintf(status,"Pre-Scanning Killfile ( %d / %d )\n",l,nkills);
							statusWindow->setText(status);
							for(int k=start;k<=last;k++) {
								if(checked_articles[k-start])
									continue;
								do_input();
								if(statusWindow->isAborted()==TRUE || running==FALSE)
									break;

								nLog("  xhdr %d\n",k);
								sprintf(buffer,"XHDR %s %d%s",header,k,newline);
								c->send_data(buffer);
								int thislen = c->read_data(hdbuffer,hdsize);
								if(*hdbuffer!='2')
									continue;

								// grab a bit more
								if(thislen < 5 || strncmp(&hdbuffer[thislen-5],"\r\n.\r\n",5)!=0) {
									char *hdptr = &hdbuffer[thislen];
									int bread = c->read_data_chunk(hdptr,hdsize - thislen);
									//bytes_downloaded += bread;
								}
								//printf("LINE: *%s*\n",hdbuffer);
								char *ptr = strchr(hdbuffer,'\n');
								if(ptr==NULL)
									continue;
								ptr++;
								ptr = strchr(ptr,' ');
								if(ptr==NULL)
									continue;
								ptr++;
								char *nextline = strstr(ptr,"\r\n");
								if(nextline != NULL)
									*nextline = '\0';

								//printf("ptr: *%s*\n",ptr);
								char *match = stristr_q(ptr,kill->text);
								BOOL matched_text = ( kill->match==0 && match!= NULL ) || ( kill->match==1 && match==NULL );
								if( matched_text ) {
									//matched text

									// kill it!
									BOOL important = FALSE;
									if(killArticle(k - start + 1,l,&important,statusWindow))
										skip_articles[k-start] = TRUE;
									if(important)
										mark_important[k-start] = TRUE;

									if(!kill->carryon) {
										nLog("skip rest\n");
										checked_articles[k-start] = TRUE;
									}
								}
							}
							if(statusWindow->isAborted()==TRUE || running==FALSE)
								break;
						}
					}
					delete [] checked_articles;
					checked_articles = NULL;
}
*/

/* Fetch news. If 'thisgroup' equals -1, then all (subscribed) groups will
 * be checked for new messages. Otherwise, it specifies the position in the
 * group list of the single group to download from.
 */
void getnews(int thisgroup) {
	nLog("getnews((int)%d) called\n",thisgroup);
	closeSockets();
	Connection *c = new Connection();
	if(!c->init()) {
		MUI_RequestA(app,0,0,"Error!","_Okay","\33cPlease start a\nTCP stack!",0);
		delete c;
		return;
	}
	char *buffer = new char[MAXLINE + 1];
	char *big_buffer = new char[big_bufsize_g + 1];
	static char windowtitle[256] = "";
		*windowtitle = '\0';
	int time1=0,time2=0;

	int entries = 0;
	getGdataEntries(&entries);
	GroupData **gdata_list = new GroupData *[entries];
	//pre-scan
	int dg=0,i=0;
	int count = 0;
	for(dg=0;dg<entries;dg++) {
		GroupData *gdata = NULL;
		getGdata(dg,&gdata);
		if(gdata == NULL)
			continue;
		gdata_list[count++] = gdata;
	}
	entries = count;

	for(dg=0;dg<entries;dg++) {
		//getGdata(dg,&gdata_list[dg]);
		GroupData *gdata = gdata_list[dg];
		//if(gdata==NULL)
		//	continue;
		if( (gdata->s && thisgroup==-1) || dg==thisgroup) {
			//set correct article pointer
			if(gdata->flags[3]==0) {
				//first time
				gdata->flags[4]=0;
				if(gdata->max_dl>100 || gdata->max_dl==-1) {
					LONG result = 0;
					if(gdata->max_dl==-1) {
						sprintf(status_buffer_g,"\33cYou have not yet downloaded news for\n%s\nDo you wish to fetch all available messages,\nor only the last 100 messages?",gdata->name);
						result = MUI_RequestA(app,0,0,"Download","_Last 100 Messages|_All Messages",status_buffer_g,0);
					}
					else {
						sprintf(status_buffer_g,"\33cYou have not yet downloaded news for\n%s\nDo you wish to fetch the last %d messages,\nor only the last 100 messages?",gdata->name,gdata->max_dl);
						sprintf(buffer,"_Last 100 Messages|L_ast %d Messages",gdata->max_dl);
						result = MUI_RequestA(app,0,0,"Download",buffer,status_buffer_g,0);
					}
					if(result==0)
						gdata->flags[4]=0;
					else
						gdata->flags[4]=-1;
				}
				gdata->flags[3]=-1;
			}
		}
	}
	GroupData *dl_this_gdata = NULL;
	if(thisgroup != -1)
		dl_this_gdata = gdata_list[thisgroup];

	// perform a sort based on servers
	sortGroupList(gdata_list,entries);
	
	/*for(dg=0;dg<entries;dg++) {
		Server *server = getServer(gdata_list[dg]->serverID);
		printf("%d %d : %s : %s\n", dg, server->ID, server->nntp, gdata_list[dg]->name);
	}
	return;*/
	
	mIDhist = NULL;
	delete_dis = TRUE;
	StatusWindow *statusWindow = new StatusWindow(app,"Downloading..");
	statusWindow->setText("\n");
	statusWindow->resize();
	Server *server = NULL;

	// killfiles
	nkills = 0;
	get(NLIST_acc_KILLFILE,MUIA_NList_Entries,&nkills);
	nLog("found %d killfiles\n",nkills);
	killfiles = new KillFile *[nkills];
	replace_killfile = new BOOL[nkills];
	killfile_in_this_group = new BOOL[nkills];
	killfile_types = new int[nkills];
	for(i=0;i<nkills;i++) {
		KillFile *okill = NULL;
		DoMethod(NLIST_acc_KILLFILE,MUIM_NList_GetEntry,i,&okill);
		killfiles[i] = new KillFile();
		nLog("  about to copy across killfile %d\n",i);
		*killfiles[i] = *okill;
		replace_killfile[i] = FALSE;
		killfile_in_this_group[i] = FALSE;

		KillFile *kill = killfiles[i];
		kill->header[63] = '\0';
		kill->text[255] = '\0';
		kill->ngroups[63] = '\0';

		int type = 0; // 0=unknown, 1=subject, 2=from, 3=date, 4=message-ID, 5=refs
		if( equals("SUBJECT:",kill->header) )
			type = 1;
		else if( equals("FROM:",kill->header) )
			type = 2;
		else if( equals("DATE:",kill->header) )
			type = 3;
		else if( equals("MESSAGE-ID:",kill->header) )
			type = 4;
		else if( equals("REFERENCES:",kill->header) )
			type = 5;
		killfile_types[i] = type;
	}

	for(dg=0;dg<entries;dg++) {
		GroupData *gdata = gdata_list[dg];

		// initialise killfiles
		for(i=0;i<nkills;i++) {
			KillFile *kill = killfiles[i];
			kill->header[63] = '\0';
			kill->text[255] = '\0';
			kill->ngroups[63] = '\0';

			killfile_in_this_group[i] = FALSE;
			if(stristr_q(gdata->name,kill->ngroups)!=NULL) {
				killfile_in_this_group[i] = TRUE;
			}
		}

		//if( (gdata->s && thisgroup==-1) || dg==thisgroup) {
		if( ( gdata->s && dl_this_gdata == NULL ) || gdata == dl_this_gdata ) {

			sprintf(windowtitle,"Downloading: %s",gdata->name);
			nLog("Downloading: %s",gdata->name);
			statusWindow->setTitle(windowtitle);

			//if(server==NULL || (server->ID != gdata->serverID && (gdata->serverID!=0 || server->def==FALSE))) {
			if(server != getServer(gdata->serverID)) {
				if(c->connected())
					hangUp(c);
				server = getServer(gdata->serverID);
				// open a socket
				sprintf(status_buffer_g,"Connecting with '%s'...\n",server->nntp);
				statusWindow->setText(status_buffer_g);
				if(!logIn(c,server,statusWindow)) {
					if(c->connected())
						hangUp(c);
					// scan to next server
					while(dg<entries) {
						dg++;
						gdata = gdata_list[dg];
						if( ( gdata->s && dl_this_gdata == NULL ) || gdata == dl_this_gdata ) {
							if(server != getServer(gdata->serverID)) {
								// this group is on a different server
								dg--; // since it will increment when we continue
								break;
							}
						}
					}
					continue;
				}
			}
			gdata->flags[0]=TRUE;
			//read this group
			sprintf(status_buffer_g,"Scanning %s\n",gdata->name);
			statusWindow->setText(status_buffer_g);

			sprintf(buffer,"group %s%s",gdata->name,newline);
			c->send_data(buffer);
			c->read_data(buffer);
			StripNewLine(buffer);

			if(statusWindow->isAborted()==TRUE || running==FALSE)
				break;
			if(*buffer!='2')	 {
				sprintf(status_buffer_g,"\33cCannot get Group information from this Newsserver\nError: %s",buffer);
				MUI_RequestA(app,0,0,"Error!","_Okay",status_buffer_g,0);
				continue;
			}

			int nidz=0;
			char cps[32]="";
			char word1[256] = "";
			char word2[256] = "";
			word(word1,buffer,3);
			word(word2,buffer,4);
			int first = atoi(word1);
			int last = atoi(word2);
			int getlast = 100;
			if(gdata->max_dl!=-1) {
				if( (gdata->max_dl>100 && gdata->flags[4]==0) || gdata->max_dl<=100)
					getlast=gdata->max_dl;
			}
			if(gdata->flags[4]==-1)
				gdata->flags[4]=last-getlast+1;
			if(gdata->flags[4]>last+1) // if flags[4]=article+1, then no new news!
				gdata->flags[4]=first;  // this test shouldn't really be needed!
			if(gdata->max_dl!=-1) {
				if(gdata->flags[4]<last-gdata->max_dl+1)
					gdata->flags[4]=last-gdata->max_dl+1;
			}
			if(gdata->flags[4]<first)
				gdata->flags[4]=first;
			if(gdata->flags[4]==last+1) {
				sprintf(status_buffer_g,"No new news for %s!\n",gdata->name);
				statusWindow->setText(status_buffer_g);
			}
			else {
				sprintf(status_buffer_g,"Downloading Articles: %d to %d\n",gdata->flags[4],last);
				statusWindow->setText(status_buffer_g);
			}
			int start = gdata->flags[4];
			//printf("%d %d %d\n",first,start,last);
			if(gdata->flags[4] <= 0)
				gdata->moreflags[3]=FALSE; // need to check for message IDs
			// download first to last (inclusive)
			int bread=0,bout=0;
			int tline=0;
			int mcachepos = 0;
			BOOL uptodate = FALSE;
			if(start!=last+1) { // only if new news!
				mIDhist = NULL;
				histlen = 0;
				// only read index if we need to check for message IDs !
				nLog("up to date with newsgroup? %d\n",gdata->moreflags[3]);
				BOOL needIDs = FALSE;
				if(gdata->moreflags[3] == FALSE)
					needIDs = TRUE;
				
				// we now switch to false, in case there is a crash / connectionloss /
				// whatever whilst downloading - that way, next time we will check for
				// duplicates as we should.
				// otherwise, the flag will be set to TRUE (if appropriate) after
				// downloading for this group.
				uptodate = gdata->moreflags[3];
				gdata->moreflags[3] = FALSE;
				account.save_data(); // so this setting is saved!

				// switch to current group
				if(account.vgroupdl!=0) // change group always
					set_and_read(gdata);

				if(needIDs || (account.flags & Account::CHECKFORDUPS) != 0) {
					sprintf(status_buffer_g,"Reading in messageID history...\n");
					statusWindow->setText(status_buffer_g);
					
					Vector * vector = NULL;
					GroupData *gdata2 = NULL;
					getGdataDisplayed(&gdata2);
					// read in ID history
					if(gdata->ID!=gdata2->ID) {
						vector = new Vector(2048);
						BOOL res = read_index(gdata,vector);
						histlen=vector->getSize();
					}
					else
						get(NLIST_messagelistdata,MUIA_NList_Entries,&histlen);
					if(histlen>0) {
						mIDhist = new char *[histlen];
						nLog("about to copy messageIDs..\n");
						for(int k=0;k<histlen;k++) {
							MessageListData * mdata = NULL;
							if(vector!=NULL)
								mdata=((MessageListData **)vector->getData())[k];
							else
								DoMethod(NLIST_messagelistdata,MUIM_NList_GetEntry,k,&mdata);
							mIDhist[k] = new char[256];
							strncpy(mIDhist[k],mdata->messageID,255);
							mIDhist[k][255] = '\0';
							//mIDhist[k] = mdata->mIDhash;
						}
					}
					else
						mIDhist = NULL;
					if(vector!=NULL) {
						nLog("about to delete vector contents..\n");
						for(i=0;i<vector->getSize();i++)
							delete vector->getData()[i];
						nLog("  done\n");
						delete vector;
					}
				}
				
				//index_handler *ih = new index_handler(gdata);
				//fetch required articles
				nidz=last-start+1;
				mcachepos=0;

				int starttime = clock();
				/*BOOL *skip_articles = new BOOL[nidz];
				BOOL *mark_important = new BOOL[nidz];
				for(int i=0;i<nidz;i++) {
					skip_articles[i] = FALSE;
					mark_important[i] = FALSE;
				}*/

				BOOL use_xover = FALSE;
				char *xover_ptr = NULL;
				if(gdata->flags[5]==TRUE)
					use_xover = TRUE;

				// compatible with killfiles for this group?
				for(int l=0;use_xover && l<nkills;l++) {
					if( killfile_in_this_group[l] && killfile_types[l] == 0 ) {
						use_xover = FALSE;
					}
				}
				nLog("use_xover = %d\n",use_xover);

				if(use_xover) {
					// killfiles - get info for any headers
					//killWithXHDR(skip_articles,mark_important,c,start,last,&statusWindow);

					sprintf(buffer,"XOVER %d-%d%s",start,last,newline);
					c->send_data(buffer);
					int thislen = c->read_data(big_buffer,big_bufsize_g);
					if(*big_buffer!='2') {
						nLog("  no xover!\n");
						use_xover = FALSE;
					}
					else {
						xover_ptr = strchr(big_buffer,'\n');
						while(xover_ptr == NULL) {
							// grab a bit more
							if(thislen < 5 || strncmp(&big_buffer[thislen-5],"\r\n.\r\n",5)!=0) {
								char *hdptr = &big_buffer[thislen];
								bread = c->read_data_chunk(hdptr,big_bufsize_g - thislen);
							}
							xover_ptr = strchr(big_buffer,'\n');
						}
						xover_ptr++;
					}
				}
					
				sprintf(status_buffer_g,"Downloading (1/%d) %s\n",nidz,cps);
				statusWindow->setText(status_buffer_g);
				for(int k=start;k<=last;k++) {
					nLog("  downloading %d\n",k);
					//printf("  downloading %d\n",k);
					int k1 = k;
					do_input();
					if(statusWindow->isAborted()==TRUE || running==FALSE)
						break;

					int bytes_downloaded=0;
					time1=clock();
					int thislen = 0;
					if(!use_xover) {					
						sprintf(buffer,"HEAD %d%s",k1,newline);
						c->send_data(buffer);
						thislen = c->read_data(big_buffer,big_bufsize_g);
						bytes_downloaded += thislen;
						if(*big_buffer!='2') {
							nLog("  no head!\n");
							gdata->flags[4] = k1+1;
							k1 = -1;
						}
					}
					int id1 = 0;
					char filename[300] = "";
					MessageListData * mdata = NULL;
					BPTR fileout = NULL;
					if(k1!=-1) {
						for(;;) { // we want a file that doesn't exist!
							sprintf(filename,"NewsCoasterData:Folder_%d/news_%d",gdata->ID,gdata->nextmID++);
							
							/*
							if(!exists(filename)) {
								fileout=Open(filename,MODE_NEWFILE);
								if(fileout!=NULL)
									break;
								else
									gdata->nextmID+=10;
							}
							else
								gdata->nextmID+=10;
							*/
							fileout=Open(filename,MODE_NEWFILE);
							if(fileout != NULL)
								break;
							else
								gdata->nextmID += 8;

						}
						id1=gdata->nextmID-1;
						nLog("got file ID %d\n",id1);
					}

					if(fileout==NULL)
						k1=-1;

					if(k1!=-1) {
						mdata = new MessageListData();
						if(mdata==NULL)  {
							printf("Error - Can't Allocate mdata Object!!!\n");
							nLog("Error - Can't Allocate mdata Object!!!\n");
							delete_dis=FALSE;
							delete c;
							for(i=0;i<nkills;i++)
								delete killfiles[i];
							delete [] killfiles;
							killfiles = NULL;
							nkills = 0;
							delete [] big_buffer;
							return;
						}
						mdata->init();
						for(i=0;i<NMFLAGS;i++)
							mdata->flags[i] = 0;
						mdata->ID=id1;
						mdata->flags[6]=gdata->ID;
						mdata->size=0;
						mdata->flags[1]=TRUE;
						if(gdata->flags[5]==TRUE)
							mdata->flags[12]=TRUE;
						else
							mdata->flags[12]=FALSE;
						mdata->flags[11] = k;
						
						/*if(mark_important[k-start])
							mdata->flags[13] = 1;*/
					}

					do_input();
					if(statusWindow->isAborted()==TRUE || running==FALSE) {
						k1 = -1;
					}

					if(k1!=-1) {
						
						if(use_xover) {
							char *nextline = strchr(xover_ptr,'\n');
							if(nextline == NULL) {
								// grab some more
								char *ptr = big_buffer;
								while(*xover_ptr != '\0') {
									*ptr = *xover_ptr;
									ptr++;
									xover_ptr++;
								}
								int clen = (int)ptr - (int)big_buffer;
								int thislen = c->read_data(ptr,big_bufsize_g - clen);
								xover_ptr = big_buffer;
								nextline = strchr(xover_ptr,'\n');
							}
							*nextline = '\0';
							//printf("LINE: *%s*\n",xover_ptr);
							
							//char *start = strchr(big_buffer,'\n');
							int len = 0;
							char *buffer1_2 = NULL;
							char *buffer1_3 = NULL;

							/*if(start == NULL) {
								//printf("NULL!\n");
								goto cont;
							}
							goto cont;
							start++;*/
							
							int aid = atoi(xover_ptr);
							if(aid != k) {
								nLog("ID %d != requested %d : skip\n",aid,k);
								*nextline = '\n';
								k1 = -1;
							}
							else
							{
								//BOOL skip = skip_articles[k-start];
								BOOL skip = FALSE;
								char *subject = NULL;
								char *from = NULL;
								char *date = NULL;
								char *messageID = NULL;
								char *references = NULL;
								if(skip)
									goto cont;

								// subject
								if( (buffer1_2 = strchr(xover_ptr,'\t')) == NULL )
									goto cont;
								buffer1_2++;
								if( (buffer1_3 = strchr(buffer1_2,'\t')) == NULL )
									goto cont;
								*buffer1_3 = '\0';
								subject = buffer1_2;
								translateIso(isobuf,buffer1_2);
								strncpy(mdata->subject,isobuf,IHEADENDSUBJECT);
								//strncpy(mdata->subject,buffer1_2,IHEADENDSUBJECT);
								mdata->subject[IHEADENDSUBJECT]= '\0';
								len = sprintf(buffer,"Subject: %s\n",buffer1_2);
								Write(fileout,buffer,len);
								mdata->size += len;
								// advert kill
								if(account.use_sizekill) {
									char *word2 = wordLast(subject);
									if(word2) {
										if(*word2 != '\0') {
											BOOL okay = TRUE;
											int w2len = strlen(word2);
											for(int l=0;l<w2len;l++) {
												if(!isdigit((int)word2[l])) {
													okay = FALSE;
													break;
												}
											}
											if(okay) {
												if(strlen(word2)>=3) { // if number is 3 digits or more
													nLog("*** Message Killed! - 'Advert' %s (%d)\n",word2,k1 - start + 1);
													sprintf(status_buffer_g,"Message %d Killed! ('Advert')\n",k1 - start + 1);
													statusWindow->setText(status_buffer_g);
													skip = TRUE;
													goto cont;
												}
											}
										}
									}
								}

								// from
								buffer1_2 = &buffer1_3[1];
								if( (buffer1_3 = strchr(buffer1_2,'\t')) == NULL )
									goto cont;
								*buffer1_3 = '\0';
								from = buffer1_2;
								translateIso(isobuf,buffer1_2);
								strncpy(mdata->from,isobuf,IHEADENDSHORT);
								//strncpy(mdata->from,buffer1_2,IHEADENDSHORT);
								mdata->from[IHEADENDSHORT]=0;
								len = sprintf(buffer,"From: %s\n",buffer1_2);
								Write(fileout,buffer,len);
								mdata->size += len;

								// date
								buffer1_2 = &buffer1_3[1];
								if( (buffer1_3 = strchr(buffer1_2,'\t')) == NULL )
									goto cont;
								*buffer1_3 = '\0';
								date = buffer1_2;
								*mdata->datec = '\0';
								DateHandler::read_date(&mdata->ds,buffer1_2,mdata->c_date,mdata->c_time);
								len = sprintf(buffer,"Date: %s\n",buffer1_2);
								Write(fileout,buffer,len);
								mdata->size += len;

								// messageID
								buffer1_2 = &buffer1_3[1];
								if( (buffer1_3 = strchr(buffer1_2,'\t')) == NULL )
									goto cont;
								*buffer1_3 = '\0';
								messageID = buffer1_2;
								strncpy(mdata->messageID,buffer1_2,IHEADENDMID);
								mdata->messageID[IHEADENDMID]=0;
								len = sprintf(buffer,"Message-ID: %s\n",buffer1_2);
								Write(fileout,buffer,len);
								mdata->size += len;
								// duplicate check
								{
									char **hist = mIDhist;
									for(int l=0;l<histlen;l++) {
										if(strcmp(*hist++,mdata->messageID)==0) {
											// already downloaded!
											nLog("*** Duplicate Message! - %s (%d, %d)\n",mdata->messageID,k1 - start + 1,l);
											sprintf(status_buffer_g,"Message %d Killed! (Duplicate Message)\n",k1 - start + 1);
											statusWindow->setText(status_buffer_g);
											skip = TRUE;
											goto cont;
										}
									}
								}

								// references
								buffer1_2 = &buffer1_3[1];
								if( (buffer1_3 = strchr(buffer1_2,'\t')) == NULL )
									goto cont;
								*buffer1_3 = '\0';
								references = buffer1_2;
								{
									char *lastref = wordLast(buffer1_2);
									strncpy(mdata->lastref,lastref,IHEADENDMID);
									mdata->lastref[IHEADENDMID]=0;
								}
								len = sprintf(buffer,"References: %s\n",buffer1_2);
								Write(fileout,buffer,len);
								mdata->size += len;

								// bytes
								buffer1_2 = &buffer1_3[1];
								if( (buffer1_3 = strchr(buffer1_2,'\t')) == NULL )
									goto cont;
								*buffer1_3 = '\0';

								// lines
								buffer1_2 = &buffer1_3[1];
								if( (buffer1_3 = strchr(buffer1_2,'\t')) == NULL )
									goto cont;
								*buffer1_3 = '\0';
								len = sprintf(buffer,"Lines: %s\n",buffer1_2);
								Write(fileout,buffer,len);
								mdata->size += len;
								mdata->flags[8] = atoi(buffer1_2);
								// kill on # of lines
								if(gdata->flags[2]>0) {
									if( mdata->flags[8] > gdata->flags[2]) {
										nLog("*** Message Killed! - %d longer than %d lines (%d)\n",mdata->flags[8],gdata->flags[2],k1 - start + 1);
										sprintf(status_buffer_g,"Message %d Killed! (Too Many Lines)\n",k1 - start + 1);
										statusWindow->setText(status_buffer_g);
										skip = TRUE;
										goto cont;
									}
								}

								cont:
								// killfile check
								for(int l=0;l<nkills;l++) {
									if(killfile_in_this_group[l]) {
										KillFile *kill = killfiles[l];
										
										int type = killfile_types[l];

										if(type == 1 && subject == NULL)
											type = 0;
										else if(type == 2 && from == NULL)
											type = 0;
										else if(type == 3 && date == NULL)
											type = 0;
										else if(type == 4 && messageID == NULL)
											type = 0;
										else if(type == 5 && references == NULL)
											type = 0;

										char *match = NULL;
										BOOL found = TRUE;
										if(type == 1)
											match = stristr_q(subject,kill->text);
										else if(type == 2)
											match = stristr_q(from,kill->text);
										else if(type == 3)
											match = stristr_q(date,kill->text);
										else if(type == 4)
											match = stristr_q(messageID,kill->text);
										else if(type == 5)
											match = stristr_q(references,kill->text);
										else if(type == 0){
											// we shouldn't be using XHDR here
											found = FALSE;
										}
										else {
											// something went wrong
											found = FALSE;
										}
										
										BOOL matched_text = ( kill->match==0 && match!= NULL ) || ( kill->match==1 && match==NULL );
										if( found && matched_text ) {
											//matched text

											// kill it!
											BOOL important = FALSE;
											if(killArticle(k1 - start + 1,l,&important,statusWindow))
												skip = TRUE;
											
											if(important)
												mdata->flags[13] = 1;

											if(!kill->carryon) {
												nLog("skip rest\n");
												break;
											}
										}
									}
								}

								if(skip) {
									k1 = -1;
								}
								// move to next line
								xover_ptr = &nextline[1];
							}

						}
						else {
							if(thislen < 5 || strncmp(&big_buffer[thislen-5],"\r\n.\r\n",5)!=0) {
								char *hdptr = &big_buffer[thislen];
								bread = c->read_data_chunk(hdptr,big_bufsize_g - thislen);
								bytes_downloaded += bread;
							}
							else {
								big_buffer[thislen-3] = 0;
								bread = -3;
							}

							stripEscapes(big_buffer);

							char *buffer1_2 = strstr(big_buffer,"\r\n") +2;
							bread -= (int)buffer1_2 - (int)big_buffer;

							Write(fileout,buffer1_2,thislen + bread);
							mdata->size += thislen + bread;

							//nLog("about to process header\n");
							on_refs = FALSE;
							for(;;) {
								char *buffer1_3 = strstr(buffer1_2,newline);
								if(buffer1_3 != NULL) {
									*buffer1_3 = '\0';
									//nLog("  buffer1_2 = %d\n",buffer1_2);
									//nLog("  buffer1_3 = %d\n",buffer1_3);
									if(processHeader(buffer1_2,gdata,mdata,gdata->name,statusWindow,k1 - start + 1)) {
										k1=-1;
										break;
									}
									buffer1_2 = &buffer1_3[2];
									if(*buffer1_2 == '\0')
										break;
								}
								else {
									big_buffer[bread] = '\0';
									//nLog("  buffer1_2     = %d\n",buffer1_2);
									//nLog("  big_buffer  = %d\n",big_buffer);
									//nLog("  bread         = %d\n",bread);
									if(processHeader(buffer1_2,gdata,mdata,gdata->name,statusWindow,k1 - start + 1)) {
										k1=-1;
										break;
									}
									break;
								}
							}
						}
					}
					if(gdata->flags[5]==TRUE && k1!=-1) {
						if((account.flags & Account::QUIETDL) != 0)
							sprintf(status_buffer_g,"Downloading (%d/%d) %s\n",k1-start+1,nidz,cps);
						else
							sprintf(status_buffer_g,"Downloading (%d/%d) %s\n%s - %s",k1-start+1,nidz,cps,mdata->from,mdata->subject);
						//sprintf(status_buffer_g,"%d\n",k1-start+1);
						statusWindow->setText(status_buffer_g);

						Write(fileout,onlineflag,onlineflag_len);
						mdata->size += onlineflag_len;
					}

					if(gdata->flags[5]==FALSE) {
						if(k1!=-1) {
							Write(fileout,"\n",1);
							mdata->size ++;
							sprintf(buffer,"BODY %d%s",k1,newline);
							c->send_data(buffer);

							if((account.flags & Account::QUIETDL) != 0)
								sprintf(status_buffer_g,"Downloading (%d/%d) %s\n",k1-start+1,nidz,cps);
							else
								sprintf(status_buffer_g,"Downloading (%d/%d) %s\n%s - %s",k1-start+1,nidz,cps,mdata->from,mdata->subject);
							statusWindow->setText(status_buffer_g);
						}
						if(k1!=-1) {
							// now read the body data
							BOOL available = TRUE;
							if(download_body(&available,c,fileout,&bread,&bout,statusWindow,FALSE,buffer) == FALSE) {
								if(!available)
									statusWindow->setText("BODY not found\n");
								k1 = -1;
							}
							bytes_downloaded += bread;
							mdata->size += bout;
						}
					}
					Close(fileout);
					fileout=NULL;

					do_input();
					if(k1==-1 || statusWindow->isAborted()==TRUE || running==FALSE) {
						nLog("  deleting file %s\n",filename);
						if(*filename!=0)
							DeleteFile(filename);
						if(mdata) {
							delete mdata;
							mdata=NULL;
						}
						nLog("  done\n");
					}
					else {
						/*if(*mdata->type==0)
							strcpy(mdata->type,"text/plain");*/
						/*GroupData * cdg = NULL;
						getGdataDisplayed(&cdg);
						if(cdg->ID==gdata->ID) {
							DoMethod(NLIST_messagelistdata,MUIM_NList_InsertSingle,mdata,MUIV_NList_Insert_Bottom);
							//setEnabled();
						}*/
						//ih->write(mdata);
						write_index_single(gdata,mdata);
						gdata->nummess++;
						gdata->num_unread++;
					}

					time2 = clock() - time1;
					if(time2 == 0)
						time2++;
					if( use_xover )
						*cps = '\0';
					else {
						sprintf(cps,"(%d cps)", (50 * bytes_downloaded) / time2 );
					}
					if(statusWindow->isAborted()==TRUE || running==FALSE)
						break;
					gdata->flags[4]=k+1;
				}
				GroupData * cdg = NULL;
				getGdataDisplayed(&cdg);
				if(cdg->ID==gdata->ID) {
					DoMethod(NLIST_messagelistdata,MUIM_NList_Sort);
					threadView();
					setEnabled();
				}
				// end this group
				//delete ih;
				//ih = NULL;
				//delete [] skip_articles;
				//skip_articles = NULL;
				//delete [] mark_important;
				//mark_important = NULL;
				
				if(use_xover) {
					int len = strlen(xover_ptr);
					// final line
					char *nextline = strchr(xover_ptr,'\n');
					if(nextline == NULL) {
						// grab some more
						/*char *ptr = big_buffer;
						while(*xover_ptr != '\0') {
							*ptr = *xover_ptr;
							ptr++;
							xover_ptr++;
						}
						int clen = (int)ptr - (int)big_buffer;
						int thislen = c->read_data(ptr,big_bufsize_g - clen);
						xover_ptr = big_buffer;
						nextline = strchr(xover_ptr,'\n');*/
						c->read_data(big_buffer,big_bufsize_g);
					}
				}
				
				/*int endtime = clock();
				int totaltime = endtime - starttime;
				printf("TIME = %d ( %fs )\n", totaltime, totaltime/50.0);*/
			}
			DateStamp(&gdata->lastdlds);
			redrawGdataAll();
			if( gdata->flags[4] > 0 &&
				( gdata->flags[4] > last || uptodate ) ) {
				// we're up to date with the newsgroup
				// so no need to check for duplicates anymore, until we
				// reset the group article pointer
				gdata->moreflags[3] = TRUE;
			}
			account.save_data(); // save the changed group information!

			if(mIDhist) {
				for(int k=0;k<histlen;k++) {
					if(mIDhist[k]) {
						delete [] mIDhist[k];
						mIDhist[k]=NULL;
					}
				}
				delete [] mIDhist;
				mIDhist=NULL;
			}
			if(statusWindow->isAborted()==TRUE || running==FALSE) {
				break;
			}
			// next newsgroup
		}
	}
	
	int enkills = 0;
	get(NLIST_acc_KILLFILE,MUIA_NList_Entries,&enkills);
	if(enkills > nkills)
		enkills = nkills;
	for(i=0;i<enkills;i++) {
		if(!replace_killfile[i])
			continue;
		KillFile *kill = killfiles[i];
		KillFile *ekill = NULL;
		DoMethod(NLIST_acc_KILLFILE,MUIM_NList_GetEntry,i,&ekill);
		// check match in case changed
		if(equals(kill->header,ekill->header) &&
			equals(kill->text,ekill->text) &&
			equals(kill->ngroups,ekill->ngroups)) {
			nLog("replacing killfile at position %d ( %s %s %s )\n",i,kill->header,kill->text,kill->ngroups);
			*ekill = *kill;
			DoMethod(NLIST_acc_KILLFILE,MUIM_NList_Redraw,i);
		}
	}
	for(i=0;i<nkills;i++)
		delete killfiles[i];
	delete [] killfiles;
	killfiles = NULL;
	delete [] replace_killfile;
	replace_killfile = NULL;
	delete [] killfile_in_this_group;
	killfile_in_this_group = NULL;
	delete [] killfile_types;
	killfile_types = NULL;
	nkills = 0;

	delete [] gdata_list;
	gdata_list = NULL;

	if(c->connected())
		hangUp(c);
	//finished all
	statusWindow->setVisible(FALSE);
	delete_dis=FALSE;

	delete c;
	delete [] big_buffer;
	delete statusWindow;
	return;
}

int shrink(BPTR file,int length,int oldsize) {
	nLog("shrink((BPTR)%d,(int)%d,(int)%d) called\n",file,length,oldsize);
	if(SetFileSize(file,length,OFFSET_BEGINNING) == -1) {
		nLog("SetFileSize() failed - pad out with extra bytes\n");
		// Alternative to shrinking the file size
		Seek(file,length,OFFSET_BEGINNING);
		for(int i=0;i<oldsize - length;i++)
			FPutC(file,'\0');
		return oldsize;
	}
	return length;
}				

BOOL getBody(BOOL *available,GroupData * gdata,MessageListData * mdata) {
	StatusWindow *statusWindow = new StatusWindow(app,"Downloading body..");
	BOOL res = getBody(available,gdata,mdata,statusWindow,-1,-1,TRUE,TRUE);
	delete statusWindow;
	return res;
}

BOOL getBody(BOOL *available,GroupData * gdata,MessageListData * mdata,StatusWindow * statusWindow,int index,int max,BOOL first, BOOL delmess) {
	nLog("getBody((GroupData *)%d,(MessageListData *)%d,(StatusWindow *)%d,(int)%d,(int)%d,(BOOL)%d,(BOOL)%d) called\n",gdata,mdata,statusWindow,index,max,first,delmess);
	Server *server = getServer(gdata->serverID);
	if(online_c != NULL && online_c->serverID != -1
		&& online_c->serverID != server->ID) {
		nLog("Different server - close sockets\n");
		closeSockets();
	}

	if(online_c == NULL) {
		online_c = new Connection();
		if(!online_c->init()) {
			MUI_RequestA(app,0,0,"Error!","_Okay","\33cPlease start a\nTCP stack!",0);
			delete online_c;
			online_c = NULL;
			return FALSE;
		}
	}
	*available = TRUE;
	//ViewWindow::sleepAll(TRUE);
	/*statusWindow->setText("\n");
	statusWindow->resize();*/
	GroupData * gdata_curr=NULL;
	BOOL OKAY = TRUE;
	char *line_buffer = new char[MAXLINE+1];
	char *big_buffer = new char[big_bufsize_g + 1];
	int bytes_downloaded=0;
	int bread = 0,bout = 0;

	int entries = 0;
	getGdataEntries(&entries);
	
	// switch to current group
	//set_and_read(gdata,TRUE);
	delete_dis=TRUE;

	char filename[300] = "";
	getFilePath(filename,gdata->ID,mdata->ID);
	BPTR lock = Lock(filename,ACCESS_READ); // READ?
	BPTR fileout = Open(filename,MODE_OLDFILE);

	LONG oldpos = 0;
	int oldsize = mdata->size;
	//BOOL xheader_zapped = FALSE;
	
	char bodyline[MAXLINE] = "";
	char command[] = "ARTICLE";
	if(strlen(mdata->messageID) >= IHEADENDMID-1) {
		NewMessage nm;
		strcpy(nm.getThisHeader,"Message-ID:");
		get_refs(&nm,gdata->ID,mdata->ID,GETREFS_NONE);
		sprintf(bodyline,"%s %s%s",command,nm.dummyHeader,newline);
	}
	else
		sprintf(bodyline,"%s %s%s",command,mdata->messageID,newline);

	set(menuitem_DISCONNECT,MUIA_Menuitem_Enabled,FALSE);
	
	BOOL connected = FALSE;
	
	int hdlen = 0;

	//while(!connected) {
	for(int attempts=0;attempts<2 && !connected;attempts++) {
		nLog("attempt number %d\n",attempts);
	
		connected = TRUE;

		if(!online_c->connected()) {
			sprintf(status_buffer_g,"Connecting with '%s'...\n",server->nntp);
			statusWindow->setText(status_buffer_g);
			if(!online_c->call_socket(server)) {
				MUI_RequestA(app,0,0,"Error!","_Okay","\33cCannot connect with Newsserver!",0);
				OKAY = FALSE;
			}
			if(OKAY) {
				online_c->read_data(line_buffer);
				StripNewLine(line_buffer);
				if(*line_buffer!='2' || statusWindow->isAborted()==TRUE) {
					if(statusWindow->isAborted()==FALSE) {
						sprintf(status_buffer_g,"\33cCannot use this Newsserver\nError: %s",line_buffer);
						MUI_RequestA(app,0,0,"Error!","_Okay",status_buffer_g,0);
					}
					online_c->send_data("quit\r\n");
					OKAY = FALSE;
				}
			}
			//authenticate
			if(OKAY && !auth(online_c,server,statusWindow)) {
				//sprintf(status_buffer_g,"\33cCannot Log Into Newsserver!\nError: %s",line_buffer);
				//MUI_RequestA(app,0,0,"Error!","_Okay",status_buffer_g,0);
				online_c->send_data("quit\r\n");
				OKAY = FALSE;
			}
			if(OKAY) {
				sprintf(line_buffer,"mode reader%s",newline);
				online_c->send_data(line_buffer);
				online_c->read_data(line_buffer);
				StripNewLine(line_buffer);
				if(statusWindow->isAborted()==FALSE) {
					if(*line_buffer!='2' && *line_buffer!='5') {
						nLog("Warning: %s received from 'mode reader' command\n",line_buffer);
					}
				}
			}
		}
		if(!OKAY) {
			nLog("not okay - exiting loop\n");
			break;
		}
	
		if(index > 0 && max > 0) {
			sprintf(status_buffer_g,"Downloading %d of %d..\n",index,max);
			statusWindow->setText(status_buffer_g);
		}
		else
			statusWindow->setText("Downloading...\n");

		statusWindow->resize();

		if(first) {
			sprintf(line_buffer,"group %s%s",gdata->name,newline);
			//printf("send: %s",line_buffer);
			online_c->send_data(line_buffer);
			online_c->read_data(line_buffer);
		}

		if(fileout!=NULL && lock!=NULL) {

			/*if(!xheader_zapped) {
				xheader_zapped = TRUE;
				Seek(fileout,-onlineflag_len,OFFSET_END);
				Read(fileout,line_buffer,onlineflag_len);
				if(strncmp(line_buffer,onlineflag,onlineflag_len)==0) {
					Seek(fileout,-onlineflag_len,OFFSET_CURRENT);
					mdata->size -= onlineflag_len;
				}
				else
					Seek(fileout,0,OFFSET_END);
				oldpos = Seek(fileout,0,OFFSET_CURRENT);
				// only set online flag afterwards, if necessary
				//mdata->flags[12]=FALSE;
				//gdata->flags[1]=TRUE;
				Write(fileout,"\n",1);
				mdata->size++;
			}*/
			hdlen = 0;
			for(;;) {
				if(!FGets(fileout,&big_buffer[hdlen],big_bufsize_g - hdlen))
					break;

				if(big_buffer[hdlen] == '\0' || big_buffer[hdlen] == '\n' || big_buffer[hdlen] == '\r')
					break;

				hdlen += strlen(&big_buffer[hdlen]);

				if(hdlen >= big_bufsize_g) {
					hdlen = big_bufsize_g;
					break;
				}
			}
			Seek(fileout,0,OFFSET_BEGINNING);
			big_buffer[hdlen] = '\0';
			mdata->size = 0;
			//printf("*%s*\n",big_buffer);

			for(int go=0;go<2;go++) {
				nLog("(%d) sending: %s\n",go,bodyline);
				online_c->send_data(bodyline);
				// now read the body data
				if(download_body(available,online_c,fileout,&bread,&bout,statusWindow,TRUE,line_buffer)) {
					// ok!
					nLog("ok!\n");
					break;
				}
				else {
					nLog("failed - available = %d\n",*available);
					if(bread <= 0) {
						nLog("lost connection?\n");
						connected = FALSE; // lost connection?
						first = TRUE;
						online_c->close();
						break;
					}
					else {
						BOOL another_go = TRUE;
						if(go==0) {
							// try with article number?
							int id = mdata->flags[11];
							nLog("trying with article number %d\n",id);
							//printf("trying with article number %d\n",id);
							if(id==0)
								another_go = FALSE;
							if( *available )
								another_go = FALSE; // don't try again if the problem was something other than BODY not available
							if(another_go) {
								sprintf(line_buffer,"group %s%s",gdata->name,newline);
								nLog("send: %s\n",line_buffer);
								//printf("send: %s",line_buffer);
								online_c->send_data(line_buffer);
								online_c->read_data(line_buffer);
								if(*line_buffer != '2') {
									nLog("group command returned %s - fail\n",line_buffer);
									another_go = FALSE;
								}
							}
							if(another_go) {
								*available = TRUE; // reset!
								sprintf(line_buffer,"stat %d%s",id,newline);
								online_c->send_data(line_buffer);
								nLog("send: %s\n",line_buffer);
								//printf("send: %s",line_buffer);
								online_c->read_data(line_buffer);
								nLog("rcvd: %s\n",line_buffer);
								//printf("rcvd: %s\n",line_buffer);
								if(*line_buffer != '2')
									another_go = FALSE;
								else {
									char word3[1024] = "";
									word(word3,line_buffer,3);
									if(stricmp(word3,mdata->messageID)!=0)
										another_go = FALSE;
								}
								if(!another_go)
									*available = FALSE; // put back again
							}
							if(another_go) {
								sprintf(bodyline,"%s %d%s",command,id,newline);
							}
						}
						if(go==1 || !another_go) {
							if( !(*available) )
								statusWindow->setText("This message is no longer available!\n");
							OKAY=FALSE;
							break;
						}
					}
				}
			}
		}
	}
	if(!connected) {
		// couldn't get connection
		OKAY = FALSE;
	}
	if(fileout!=NULL && lock!=NULL) {
		bytes_downloaded += bread;
		mdata->size += bout;
		if( statusWindow->isAborted()==TRUE || running==FALSE || OKAY==FALSE ) {
			nLog("rewrite original header\n");
		//if( FALSE ) {
			/*if( xheader_zapped ) {
				Flush(fileout);
				Seek(fileout,oldpos,OFFSET_BEGINNING);
				Write(fileout,onlineflag,onlineflag_len);
				mdata->size = oldpos + onlineflag_len;
				// flags[12] should still be TRUE
				//mdata->flags[12]=TRUE;
			}*/
			Flush(fileout);
			Seek(fileout,0,OFFSET_BEGINNING);
			if(hdlen > 0) {
				Write(fileout,big_buffer,hdlen);
			}
			if(mdata->size > hdlen) {
				mdata->size = shrink(fileout,hdlen,mdata->size);
			}
			mdata->size = hdlen;
			nLog("done\n");
		}
		else {
			mdata->flags[12]=FALSE;
			gdata->flags[1]=TRUE;
			if(mdata->size < oldsize) {
				// small bodies - we need to overwrite the 'onlineflag' text
				nLog("small body case\n");
				/*if(SetFileSize(fileout,mdata->size,OFFSET_BEGINNING) == -1) {
					nLog("SetFileSize() failed - pad out with extra bytes\n");
					// Alternative to shrinking the file size
					for(int i=0;i<oldsize - mdata->size;i++)
						FPutC(fileout,'\0');
					mdata->size = oldsize;
				}*/
				mdata->size = shrink(fileout,mdata->size,oldsize);
			}
		}
		Close(fileout);
		fileout=NULL;
		UnLock(lock);
		lock=NULL;
	}
	else {
		//printf("Can't Write To File: %s\n",filename);
		nLog("Can't Write To File: %s\n",filename);
		OKAY=FALSE;
		sprintf(line_buffer,"\33cUnable to open file:\n%s",filename);
		MUI_RequestA(app,0,0,"Error","_Okay",line_buffer,0);
	}
	if(max == -1)
		redrawGdataAll();
	
	//finished all
	delete_dis=FALSE;
	if(statusWindow->isAborted()==TRUE || running==FALSE || OKAY==FALSE) {
		closeSockets();
	}
	else {
		set(menuitem_DISCONNECT,MUIA_Menuitem_Enabled,TRUE);
		online_c->doOnlineBlocking(FALSE,FALSE);
	}
	//ViewWindow::sleepAll(FALSE);

	nLog("  available = %d\n",*available);
	if( !(*available) ) {
		if(delmess && (account.flags & Account::NODELONLINE)==0) {
			// delete message
			/*GroupData *gdata2 = NULL;
			getGdataDisplayed(&gdata2);
			if(gdata->ID==gdata2->ID) {
			}
			*/
			set_and_read(gdata);
			clearThreadView();

			nLog(" >>> deleting message header\n");
			delete_mess(gdata,mdata,FALSE,FALSE); // close==FALSE - we don't want to delete this window!

			threadView();
		}
	}

	delete [] big_buffer;
	delete [] line_buffer;

	nLog("OKAY = %d\n",OKAY);
	return OKAY;
}

void closeSockets() {
	nLog("closeSockets() called\n");
	set(menuitem_DISCONNECT,MUIA_Menuitem_Enabled,FALSE);
	if(online_c != NULL) {
		hangUp(online_c);
		delete online_c;
		online_c = NULL;
	}
}