/*
    NewsCoaster
    Copyright (C) 1999-2002 Mark Harman

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

    NewsCoaster Homepage :
        http://newscoaster.tripod.com/

    Mailing List to hear about announcements of new releases etc at:
        http://groups.yahoo.com/group/newscoaster-announce/
*/

#include "mui_headers.h"
#include <mui/NListview_mcc.h>

#include <proto/socket.h>

#include <time.h>
#include <netdb.h>
#include <string.h>
#include <stdio.h>
#include <ctype.h>

#include "Vector.h"
#include "main.h"
#include "various.h"
#include "WriteWindow.h"
#include "Indices.h"
#include "misc.h"
#include "DateHandler.h"
#include "StatusWindow.h"
#include "Connection.h"
#include "netstuff.h"
#include "strings.h"
#include "ViewWindow.h"
#include "lists.h"

int onlineflag_len = strlen(onlineflag);

static char newline[3] = {13,10,0};

Connection *online_c = NULL;

void initNetstuff() {
	SocketBase=NULL;
}

/* Authenticate to the newserver 'server' through connection 'c'.
 */
int auth(Connection *c,Server *server,char * buffer,StatusWindow * statusWindow) {
	nLog("auth((Connection *)%d,(Server *)%d,(char *)%s,(StatusWindow *)statusWindow) called\n",c,server,buffer,statusWindow);
	if(server->nntp_auth) {
		statusWindow->setText("Authenticating..\n");
		sprintf(buffer,"AUTHINFO USER %s%s",server->user,newline);
		c->send_data(buffer);
		c->read_data(buffer);
		StripNewLine(buffer);
		if(*buffer!='3') {
			nLog("USER Error: %s\n",buffer);
			MUI_RequestA(app,0,0,"Error!","_Okay","\33Authentication Failed - Unknown User",0);
			return FALSE;
		}
		sprintf(buffer,"AUTHINFO PASS %s%s",server->password,newline);
		c->send_data(buffer);
		c->read_data(buffer);
		StripNewLine(buffer);
		if(*buffer=='5') {
			nLog("PASS Error: %s\n",buffer);
			MUI_RequestA(app,0,0,"Error!","_Okay","\33Authentication Failed - Incorrect Password",0);
			return FALSE;
		}
		statusWindow->setText("In!\n");
	}
	// no call to doOnlineBlocking() !
	return TRUE;
}

/* Downloads a newsgroup listing from 'server'.
 */
int getgrouplist(Server *server) {
	nLog("getgrouplist((Server *)%d) called\n",server);
	closeSockets();
	Connection *c = new Connection();
	if(!c->init()) {
		MUI_RequestA(app,0,0,"Error!","_Okay","\33cPlease start a\nTCP stack!",0);
		delete c;
		return -1;
	}
	const int bufsize = 4096;
	char buffer[bufsize+1] = "";
	char status[256] = "";
	char temp[256] = "";
	char filename[300] = "";
	sprintf(filename,"NewsCoasterData:%s.gl",server->nntp);
	// open a socket
	StatusWindow statusWindow(app,"Downloading Group List");
	sprintf(status,"Connecting with '%s'...\n",server->nntp);
	statusWindow.setText(status);
	if(!c->call_socket(server->nntp,server->port)) {
		MUI_RequestA(app,0,0,"Error!","_Okay","\33cCannot connect with Newsserver!",0);
		delete c;
		return -1;
	}
	statusWindow.setText("Connected!\n");
	c->read_data(buffer);
	StripNewLine(buffer);
	if(*buffer!='2') {
		sprintf(temp,"\33cCannot use this Newsserver!\nError: %s",buffer);
		MUI_RequestA(app,0,0,"Error!","_Okay",temp,0);
		sprintf(buffer,"quit%s",newline);
		c->send_data(buffer);
		delete c;
		return -1;
	}
	//authenticate
	if(!auth(c,server,buffer,&statusWindow)) {
		sprintf(temp,"\33cCannot Log Into Newsserver!\nError: %s",buffer);
		MUI_RequestA(app,0,0,"Error!","_Okay",temp,0);
		sprintf(buffer,"quit%s",newline);
		c->send_data(buffer);
		delete c;
		return -1;
	}
	sprintf(buffer,"DATE%s",newline);
	c->send_data(buffer);
	c->read_data(buffer);
	StripNewLine(buffer);
	if(strncmp(buffer,"111",3)!=0) {
		nLog("Warning! - Cannot get date from newsserver!\n%s\n''Get New Newsgroups' won't work!\n",buffer);
	}
	else {
		char word1[256] = "";
		word(word1,buffer,2);
		if(*word1 != NULL) {
			strncpy(server->lastGotGroups,word1,14);
			server->lastGotGroups[14] = '\0';
			account.save_data();
			nLog("DATE: *%s*\n",server->lastGotGroups);
		}
		else {
			nLog("Warning! - Cannot get date from newsserver!\n%s\n''Get New Newsgroups' won't work!\n",buffer);
		}
	}
	
	sprintf(buffer,"LIST%s",newline);
	c->send_data(buffer);
	c->read_data_line(buffer);
	StripNewLine(buffer);
	if(*buffer!='2') {
		sprintf(temp,"\33cCannot Obtain GroupList!\nError: %s",buffer);
		MUI_RequestA(app,0,0,"Error!","_Okay",temp,0);
		sprintf(buffer,"quit%s",newline);
		c->send_data(buffer);
		delete c;
		return -1;
	}
	DeleteFile(filename);
	BPTR file = Open(filename,MODE_NEWFILE);
	delete_dis = TRUE;
	int found = 0;
	if(file) {
		int len = 0;
		do {
			len = c->read_data_chunk(&buffer[8],bufsize-9);
			// count groups
			char *str = buffer+7;
			while(str = strchr(str+1,'\n'))
				found++;
			Write(file,&buffer[8],len);
			sprintf(temp,"Found %d Groups...\n",found);
			statusWindow.setText(temp);
			do_input();
			if(statusWindow.isAborted()==TRUE || running==FALSE)
				break;
			for(int l=3;l<8;l++)
				buffer[l] = buffer[bufsize-9+l];
		} while(len == bufsize-9);
		Close(file);
		file=NULL;
	}
	delete_dis=FALSE;
	sprintf(buffer,"quit%s",newline);
	c->send_data(buffer);
	delete c;
	return found;
}

/* Download new newsgroups from 'server'.
 */
BOOL getnewgroups(Server *server) {
	nLog("getnewgroups((Server *)%d) called\n",server);
	char date[15]="";
	strncpy(date,server->lastGotGroups,14);
	date[14] = '\0';
	nLog("Date *%s*\n",date);
	char status[256] = "";
	char temp[256] = "";
	BOOL okay = TRUE;
	if(strlen(date)!=14)
		okay=FALSE;
	else {
		for(int k=0;k<14;k++) {
			if(isdigit(date[k])==FALSE) {
				okay=FALSE;
				break;
			}
		}
	}
	if(!okay) {
		sprintf(temp,"\33cDate of last newsgroupslist download not available!\nPlease try 'Fetch Group List' first.\n");
		MUI_RequestA(app,0,0,"Error!","_Okay",temp,0);
		return FALSE;
	}
	closeSockets();
	Connection *c = new Connection();
	if(!c->init()) {
		MUI_RequestA(app,0,0,"Error!","_Okay","\33cPlease start a\nTCP stack!",0);
		delete c;
		return -1;
	}
	const int bufsize = 4096;
	char buffer[bufsize+1] = "";
	char filename[300] = "";
	sprintf(filename,"NewsCoasterData:%s.gl",server->nntp);
	
	StatusWindow statusWindow(app,"Downloading New Newsgroups..");
	sprintf(status,"Connecting with '%s'...\n",server->nntp);
	statusWindow.setText(status);
	if(!c->call_socket(server->nntp,server->port)) {
		MUI_RequestA(app,0,0,"Error!","_Okay","\33cCannot connect with Newsserver!",0);
		delete c;
		return FALSE;
	}
	statusWindow.setText("Connected!\n");
	c->read_data(buffer);
	StripNewLine(buffer);
	if(*buffer!='2') {
		sprintf(temp,"\33cCannot use this Newsserver!\nError: %s",buffer);
		MUI_RequestA(app,0,0,"Error!","_Okay",temp,0);
		sprintf(buffer,"quit%s",newline);
		c->send_data(buffer);
		delete c;
		return FALSE;
	}
	//authenticate
	if(!auth(c,server,buffer,&statusWindow)) {
		sprintf(temp,"\33cCannot Log Into Newsserver!\nError: %s",buffer);
		MUI_RequestA(app,0,0,"Error!","_Okay",temp,0);
		sprintf(buffer,"quit%s",newline);
		c->send_data(buffer);
		delete c;
		return FALSE;
	}
	char newdate[16]="";
	strncpy(newdate,&date[2],6);
	newdate[6]=' ';
	strncpy(&newdate[7],&date[8],6);
	newdate[13]=0;
	
	sprintf(buffer,"DATE%s",newline);
	c->send_data(buffer);
	c->read_data(buffer);
	StripNewLine(buffer);
	if(strncmp(buffer,"111",3)!=0) {
		nLog("Warning! - Cannot get date from newsserver!\n%s\n''Get New Newsgroups' won't work!\n",buffer);
	}
	else {
		char word1[256] = "";
		word(word1,buffer,2);
		if(*word1 != NULL)
		{
			strncpy(server->lastGotGroups,word1,14);
			server->lastGotGroups[14] = '\0';
			account.save_data();
		}
		else {
			nLog("Warning! - Cannot get date from newsserver!\n%s\n''Get New Newsgroups' won't work!\n",buffer);
		}
	}
	
	sprintf(buffer,"NEWGROUPS %s%s",newdate,newline);
	c->send_data(buffer);
	c->read_data_line(buffer);
	StripNewLine(buffer);
	if(*buffer!='2') {
		sprintf(temp,"\33cCannot Obtain New NewsGroups List!\nError: %s",buffer);
		MUI_RequestA(app,0,0,"Error!","_Okay",temp,0);
		sprintf(buffer,"quit%s",newline);
		c->send_data(buffer);
		delete c;
		return FALSE;
	}
	BPTR lock=Lock(filename,ACCESS_READ);
	BPTR file=Open(filename,MODE_OLDFILE);
	delete_dis=TRUE;
	if(file) {
		int found=0;
		Seek(file,0,OFFSET_END);
		int filepos = Seek(file,0,OFFSET_CURRENT);
		Seek(file,0,OFFSET_END);
		strncpy(&buffer[6],newline,2);
		int len = 0;
		do {
			DoMethod(app,MUIM_Application_InputBuffered);
			len = c->read_data_chunk(&buffer[8],bufsize-9);
			// count groups
			char *str = &buffer[7];
			while(str = strchr(str+1,'\n'))
				found++;
			Write(file,&buffer[8],len);
			sprintf(temp,"Found %d New Groups...\n",found);
			statusWindow.setText(temp);
			for(int l=3;l<8;l++)
				buffer[l] = buffer[bufsize-9+l];
		} while(len==bufsize-9);
		if(found==0) {
			sprintf(temp,"\33cFound %d New Groups!\n",found);
			MUI_RequestA(app,0,0,"Get New Groups","_Okay",temp,0);
		}
		Close(file);
		file=NULL;
		if(found>0) {
			groupman_server = server;
			//readgrouplist(server,NLIST_groupman,filepos);
			readgrouplist(server,filepos);
			set(wnd_groupman,MUIA_Window_Open,TRUE);
		}
	}
	delete_dis=FALSE;
	if(lock) {
		UnLock(lock);
		lock=NULL;
	}
	sprintf(buffer,"quit%s",newline);
	c->send_data(buffer);
	delete c;
	return TRUE;
}

/* Post outgoing messages.
 * TODO: this doesn't really cope well with messages that are to be both
 * posted and emailed, if one of those fails; at the moment in such cases,
 * the message remains in outgoing, and will be both posted/emailed next
 * time (ie, even though one of those has already been done).
 */
void postnews(int type) {
	// type: 0=normal, 1=post online (immediate) only (flags[2]==TRUE)
	nLog("postnews((int)%d) called\n",type);
	closeSockets();
	Connection *c = new Connection();
	if(!c->init()) {
		MUI_RequestA(app,0,0,"Error!","_Okay","\33cPlease start a\nTCP stack!",0);
		delete c;
		return;
	}
	Server *server = getPostingServer();
	BPTR fileout=NULL;
	char buffer[MAXLINE] = "";
	char temp[256] = "";
	char filename[300] = "";
	char status[256]="";
	GroupData * gdata=NULL;
	GroupData * gdata_sent=NULL;
	get_gdata(&gdata,".outgoing");
	get_gdata(&gdata_sent,".sent");
	set_and_read(gdata,TRUE);
	setGdataQuiet(TRUE);
	set(NLIST_messagelistdata,MUIA_NList_Quiet,TRUE);
	BOOL newsup=FALSE,emailup=FALSE;
	//pre-scan
	int entries = 0;
	get(NLIST_messagelistdata,MUIA_NList_Entries,&entries);
	BOOL * news_sent = new BOOL[entries];
	BOOL * email_sent = new BOOL[entries];
	for(int k=0;k<entries;k++) {
		MessageListData * mdata = NULL;
		DoMethod(NLIST_messagelistdata,MUIM_NList_GetEntry,k,&mdata);
		mdata->flags[3] = mdata->flags[2];
		mdata->flags[2] = FALSE; // need to unset post-immediately flag in case we fail too early!
		if( (mdata->flags[0]==FALSE && type==0) || (mdata->flags[3]==TRUE && type==1) ) {
			if(mdata->flags[4]==TRUE)
				newsup=TRUE;
			if(mdata->flags[5]==TRUE)
				emailup=TRUE;
		}
		news_sent[k]=FALSE;
		email_sent[k]=FALSE;
	}
	if(newsup) {
		BOOL cont = TRUE;
		// open a socket to newsserver
		StatusWindow statusWindow(app,"Sending News..");
		sprintf(status,"Connecting with '%s'...\n",server->nntp);
		statusWindow.setText(status);
		if(!c->call_socket(server->nntp,server->port)) {
			MUI_RequestA(app,0,0,"Error!","_Okay","\33cCannot connect with Newsserver!",0);
			cont=FALSE;
		}
		if(cont) {
			statusWindow.setText("Connected!\n");
			c->read_data(buffer);
			StripNewLine(buffer);
			if(*buffer!='2') {
				sprintf(temp,"\33cCannot use this Newsserver!\nError: %s",buffer);
				MUI_RequestA(app,0,0,"Error!","_Okay",temp,0);
				cont=FALSE;
			}
			if(strncmp(buffer,"201",3)==0) {
				sprintf(temp,"\33cCannot post to this Newsserver!\nError: %s",buffer);
				MUI_RequestA(app,0,0,"Error!","_Okay",temp,0);
				cont=FALSE;
			}
		}
		if(cont) {
			//authenticate
			if(!auth(c,server,buffer,&statusWindow)) {
				sprintf(temp,"\33cCannot Log Into Newsserver!\nError: %s",buffer);
				MUI_RequestA(app,0,0,"Error!","_Okay",temp,0);
				cont=FALSE;
			}
		}
		//post messages
		delete_dis=TRUE;
		if(cont) {
			for(int k=0;k<entries;k++) {
				MessageListData * mdata = NULL;
				DoMethod(NLIST_messagelistdata,MUIM_NList_GetEntry,k,&mdata);
				BOOL post=FALSE;
				if(type==0 && mdata->flags[0]==FALSE)
					post=TRUE;
				else if(type==1 && mdata->flags[0]==FALSE && mdata->flags[3]==TRUE)
					post=TRUE;
				if(mdata->flags[4]==FALSE)
					post=FALSE;
				if(post) {
					sprintf(filename,"NewsCoasterData:outgoing/news_%d",mdata->ID);
					sprintf(buffer,"Sending: %s\n",mdata->subject);
					statusWindow.setText(buffer);
					fileout=Open(filename,MODE_OLDFILE);
					if(fileout) {
						sprintf(buffer,"POST%s",newline);
						c->send_data(buffer);
						c->read_data(buffer);
						StripNewLine(buffer);
						if(*buffer!='3' && *buffer!='2' && *buffer!='4') {
							MUI_RequestA(app,0,0,"Error!","_Okay","\33cCannot post to this Newsserver!",0);
							Close(fileout);
							fileout=NULL;
							cont=FALSE;
							break;
						}
						else {
							char *eof=FGets(fileout,buffer,MAXLINE);
							while(eof) {
								StripNewLine(buffer);
								do_input(); // input loop
								if(statusWindow.isAborted()==TRUE || running==FALSE)
									break;
								sprintf(buffer,"%s%s",buffer,newline);
								if(*buffer=='.')
									c->send_data(".");
								c->send_data(buffer);
								eof=FGets(fileout,buffer,MAXLINE);
							}
							Close(fileout);
							fileout=NULL;
							if(statusWindow.isAborted()==FALSE && running==TRUE) {
								sprintf(buffer,"%s.%s",newline,newline);
								c->send_data(buffer);
								c->read_data(buffer);
								StripNewLine(buffer);
								if(*buffer!='2') {
									sprintf(temp,"Error: %s",buffer);
									MUI_RequestA(app,0,0,"Error - Post Failed!","_Okay",buffer,0);
								}
								else {
									statusWindow.setText("Done\n");
									news_sent[k]=TRUE;
								}
							}
							else
								statusWindow.setText("Aborted!\n");
						}
					}
					else {
						printf("Error! Cannot open file: %s\n",filename);
						sprintf(temp,"Error! Cannot open file:\n  %s",filename);
						statusWindow.setText(temp);
					}
				}
				if(statusWindow.isAborted()==TRUE || running==FALSE)
					break;
			}
		}
		sprintf(buffer,"quit%s",newline);
		c->send_data(buffer);
		c->close();
	}
	
	if(emailup) {
		BOOL cont = TRUE;
		// open a socket to smtp server
		StatusWindow statusWindow(app,"Sending Email..");
		sprintf(status,"Connecting with '%s'...\n",account.smtp);
		statusWindow.setText(status);
		if(c->connected()) {
			sprintf(buffer,"quit%s",newline);
			c->send_data(buffer);
			c->close();
		}
		if(!c->call_socket(account.smtp,25)) {
			MUI_RequestA(app,0,0,"Error!","_Okay","\33cCannot connect with SMTP server!",0);
			cont=FALSE;
		}
		if(cont) {
			statusWindow.setText("Connected!\n");
			c->read_data(buffer);
			nLog("  received: %s\n",buffer);
			StripNewLine(buffer);
			if(*buffer!='2') {
				sprintf(temp,"\33cCannot use this SMTP server!\nError: %s",buffer);
				MUI_RequestA(app,0,0,"Error!","_Okay",temp,0);
				cont=FALSE;
			}
		}
		if(cont) {
			char *remail = ( *account.realemail != '\0' ) ? account.realemail : account.email;
			char *ptr = strchr(remail,'@');
			if(ptr != NULL)
				remail = &ptr[1];
			sprintf(buffer,"HELO %s%s",remail,newline);
			nLog("  sending: %s",buffer);
			c->send_data(buffer);
			c->read_data(buffer);
			nLog("  received: %s\n",buffer);
			StripNewLine(buffer);
			if(*buffer != '2') {
				sprintf(buffer,"HELO %s",remail);
				char *ptr = strchr(buffer,'@');
				if(ptr != NULL) {
					// try again
					strcpy(ptr,newline);
					nLog("  sending: %s",buffer);
					c->send_data(buffer);
					c->read_data(buffer);
					nLog("  received: %s\n",buffer);
					StripNewLine(buffer);
				}
				if(*buffer != '2') {
					// try yet again
					sprintf(buffer,"HELO%s",newline);
					nLog("  sending: %s",buffer);
					c->send_data(buffer);
					c->read_data(buffer);
					nLog("  received: %s\n",buffer);
					StripNewLine(buffer);
				}
				if(*buffer != '2') {
					// failed
					sprintf(temp,"\33cCannot send to this SMTP server!\nError: %s",buffer);
					MUI_RequestA(app,0,0,"Error!","_Okay",temp,0);
					cont=FALSE;
				}
			}
			for(int k=0;cont && k<entries;k++) {
				MessageListData * mdata = NULL;
				DoMethod(NLIST_messagelistdata,MUIM_NList_GetEntry,k,&mdata);
				BOOL post = FALSE;
				if(type==0 && mdata->flags[0]==FALSE)
					post = TRUE;
				else if(type==1 && mdata->flags[0]==FALSE && mdata->flags[3]==TRUE)
					post = TRUE;
				if(mdata->flags[5]==FALSE)
					post = FALSE;
				if(post) {
					sprintf(filename,"NewsCoasterData:outgoing/news_%d",mdata->ID);
					sprintf(buffer,"Sending: %s\n",mdata->subject);
					statusWindow.setText(buffer);
					fileout=Open(filename,MODE_OLDFILE);
					if(fileout) {
						NewMessage newmessage;
						get_refs(&newmessage,gdata,mdata,GETREFS_NONE);
						if(*account.realemail!=0)
							sprintf(buffer,"MAIL FROM: <%s>%s",account.realemail,newline);
						else
							sprintf(buffer,"MAIL FROM: <%s>%s",account.email,newline);
						nLog("  sending: %s",buffer);
						c->send_data(buffer);
						c->read_data(buffer);
						nLog("  received: %s\n",buffer);
						StripNewLine(buffer);
						if(*buffer!='2') {
							sprintf(temp,"\33cCannot send to this SMTP server!\nError: %s",buffer);
							MUI_RequestA(app,0,0,"Error!","_Okay",temp,0);
							Close(fileout);
							fileout=NULL;
							cont=FALSE;
							break;
						}
						char buffer2[256] = "";
						int rcpt_no=1;
						while(cont) {
							char rcpt[256] = "";
							word(rcpt,newmessage.to,rcpt_no,',');
							if(*rcpt == NULL)
								break;
							get_email(buffer2,rcpt,GETEMAIL_EMAIL);
							sprintf(buffer,"RCPT TO: <%s>%s",buffer2,newline);
							nLog("  sending: %s",buffer);
							c->send_data(buffer);
							c->read_data(buffer);
							nLog("  received: %s\n",buffer);
							StripNewLine(buffer);
							if(*buffer!='2') {
								/*sprintf(status,"Error for recipient %s: %s\n",rcpt,buffer);
								statusWindow.setText(status);*/
								sprintf(temp,"\33cError for recipient %s:\n%s",rcpt,buffer);
								MUI_RequestA(app,0,0,"Error!","_Okay",temp,0);
								Close(fileout);
								fileout=NULL;
								cont=FALSE;
								break;
							}
							rcpt_no++;
						}
						if(cont) {
							sprintf(buffer,"DATA%s",newline);
							c->send_data(buffer);
							c->read_data(buffer);
							StripNewLine(buffer);
							char *eof=FGets(fileout,buffer,MAXLINE);
							while(eof) {
								StripNewLine(buffer);
								do_input(); // input loop
								if(statusWindow.isAborted()==TRUE || running==FALSE)
									break;
								sprintf(buffer,"%s%s",buffer,newline);
								if(*buffer=='.')
									c->send_data(".");
								c->send_data(buffer);
								eof=FGets(fileout,buffer,MAXLINE);
							}
							Close(fileout);
							fileout=NULL;
							if(statusWindow.isAborted()==FALSE && running==TRUE) {
								sprintf(buffer,"%s.%s",newline,newline);
								c->send_data(buffer);
								c->read_data(buffer);
								StripNewLine(buffer);
								if(*buffer!='2') {
									nLog("Email Failed: %s\n",buffer);
									sprintf(status,"Email Failed:\n%s",buffer);
									statusWindow.setText(status);
								}
								else {
									statusWindow.setText("Done\n");
									email_sent[k]=TRUE;
								}
							}
							else {
								statusWindow.setText("Aborted!\n");
							}
						}
					}
					else {
						sprintf(temp,"Error! Cannot open file:\n  %s",filename);
						statusWindow.setText(temp);
					}
				}
				if(statusWindow.isAborted()==TRUE || running==FALSE)
					break;
			}
		}
		sprintf(buffer,"quit%s",newline);
		c->send_data(buffer);
		c->close();
	}
	
	// copy sent messages to .sent
	if(entries>0) {
		MessageListData ** mdata_sent = new MessageListData *[entries];
		int sent=0;
		for(int k=0;k<entries;k++) {
			MessageListData * mdata = NULL;
			DoMethod(NLIST_messagelistdata,MUIM_NList_GetEntry,k,&mdata);
			if(mdata->flags[4]==TRUE || mdata->flags[5]==TRUE) {
				// only move to .sent if it has both posted/emailed as required
				if( (news_sent[k]==TRUE || mdata->flags[4]==FALSE) && (email_sent[k]==TRUE || mdata->flags[5]==FALSE) )
					mdata_sent[sent++]=mdata;
				// if one of emailing/posting was successful, turn off so as to not resend
				if(news_sent[k]==TRUE && mdata->flags[4]==TRUE)
					mdata->flags[4]=FALSE;
				if(email_sent[k]==TRUE && mdata->flags[5]==TRUE)
					mdata->flags[5]=FALSE;
			}
		}
		if(sent>0) {
			StatusWindow *statusWindow = new StatusWindow(app,"Moving to sent..");
			set_and_read(gdata,TRUE);
			set(wnd_main,MUIA_Window_Sleep,TRUE);
			WriteWindow::sleepAll(TRUE);
			for(int k=0;k<sent;k++) {
				sprintf(temp,"Moving %d to sent..\n",k+1);
				statusWindow->setText(temp);
				move(gdata,gdata_sent,mdata_sent[k],0,FALSE,FALSE);
			}
			write_index(0);
			read_index(FALSE);
			set(wnd_main,MUIA_Window_Sleep,FALSE);
			WriteWindow::sleepAll(FALSE);
			delete statusWindow;
		}
		if(mdata_sent) {
			delete [] mdata_sent;
			mdata_sent=NULL;
		}
	}
	delete_dis=FALSE;
	setGdataQuiet(FALSE);
	set(NLIST_messagelistdata,MUIA_NList_Quiet,FALSE);
	if(news_sent) {
		delete [] news_sent;
		news_sent=NULL;
	}
	if(email_sent) {
		delete [] email_sent;
		email_sent=NULL;
	}
	delete c;
	return;
}

/* Being lazy and using some globals.. */
int        histlen          = 0;
char     **mIDhist          = NULL;
int        nkills           = 0;
KillFile **killfiles        = NULL;
BOOL      *replace_killfile = NULL;
BOOL       on_refs          = FALSE;

/* Process the line of header in 'buffer'.
 */
int processHeader(char * buffer,GroupData * gdata,MessageListData * mdata,char * ngroup,StatusWindow *statusWindow,int article) {
	nLog("processHeader((char *)%s,...,(MessageListData *)%d,...,(int)%d) called\n",buffer,mdata,article);
	int skip = FALSE;
	char word1[4096] = "";
	char status[256] = "";
	int w1len = wordFirstAndLenUpper(word1,buffer);
	//killfile
	for(int l=0;l<nkills;l++) {
		KillFile *kill = killfiles[l];
		kill->header[63] = '\0';
		kill->text[255] = '\0';
		kill->ngroups[63] = '\0';

		if(*kill->header=='\0' || equals(word1,kill->header)) {
			//matched header
			char *match = stristr_q(&buffer[w1len+1],kill->text);
			//if(stristr_q(&buffer[w1len+1],kill->text)) {
			if( (kill->match==0 && match!= NULL) || (kill->match==1 && match==NULL) ) {
				//matched text
				if(stristr_q(ngroup,kill->ngroups)!=NULL) {
					//matched group
					// kill it!
					nLog("*** Message Killfile match! - %s %s %s (%d, %d)\n",kill->header,kill->text,kill->ngroups,article,l);
					nLog("    match = %d ; action = %d\n",kill->match,kill->action);
					if(kill->action==0) {
						skip = TRUE;
						sprintf(status,"Message %d Killed! (Killfile)\n",article);
						statusWindow->setText(status);
					}
					else if(kill->action==1)
						mdata->flags[13] = 1;
					if(!replace_killfile[l]) {
						KillFile * newkill = new KillFile();
						strncpy(newkill->header,kill->header,63);
						strncpy(newkill->text,kill->text,255);
						strncpy(newkill->ngroups,kill->ngroups,63);
						newkill->header[63] = '\0';
						newkill->text[255] = '\0';
						newkill->ngroups[63] = '\0';
						newkill->ds = kill->ds;
						// lastused is at current datetime now - see KillFile constructor
						newkill->expiretype = kill->expiretype;
						newkill->expire = kill->expire;
						//newkill->type = kill->type;
						newkill->match = kill->match;
						newkill->action = kill->action;
						newkill->carryon = kill->carryon;
						nLog("  about to replace killfile..\n");
						// copy it back
						*kill = *newkill;
						replace_killfile[l] = TRUE;
						delete newkill;
					}
					nLog("  done\n");
					if(!kill->carryon) {
						nLog("skip rest\n");
						break;
					}
				}
			}
		}
	}
	nLog("finished kills\n");
	
	
	if(!skip) {
		// cope with multiple line references
		if(on_refs) {
			nLog("on_refs\n");
			if(strchr(word1,':') != NULL)
				on_refs = FALSE;
			else {
				char *lastref = NULL;
				char *ptr = buffer;
				for(;;) {
					ptr = strchr(ptr,'<');
					if(ptr == NULL)
						break;
					lastref = ptr;
					ptr++;
				}
				if(lastref != NULL) {
					nLog("lastref: %s\n",lastref);
					int i=0;
					while(*lastref != NULL) {
						mdata->lastref[i] = *lastref;
						i++;
						if(*lastref=='>' || i==IHEADENDMID)
							break;
						lastref++;
					}
					mdata->lastref[i] = '\0';
				}
			}
		}
		nLog("about to scan : %s\n",word1);

		if(strcmp(word1,"FROM:")==0)
		{
			strncpy(mdata->from,&buffer[6],IHEADENDSHORT);
			mdata->from[IHEADENDSHORT]=0;
		}
		else if(strcmp(word1,"NEWSGROUPS:")==0)
		{
			strncpy(mdata->newsgroups,&buffer[12],IHEADENDSHORT);
			mdata->newsgroups[IHEADENDSHORT]=0;
		}
		else if(strcmp(word1,"SUBJECT:")==0)
		{
			mdata->subject[0]= '\0';
			strncpy(mdata->subject,&buffer[9],IHEADEND);
			mdata->subject[IHEADEND]= '\0';
			if(account.use_sizekill) {
				char *word2=wordLast(mdata->subject);
				if(word2) {
					if(*word2!=0) {
						BOOL okay=TRUE;
						int w2len = strlen(word2);
						for(int l=0;l<w2len;l++) {
							if(!isdigit((int)word2[l])) {
								okay=FALSE;
								break;
							}
						}
						if(okay) {
							if(strlen(word2)>=3) { // if number is 3 digits or more
								nLog("*** Message Killed! - 'Advert' %s (%d)\n",word2,article);
								sprintf(status,"Message %d Killed! ('Advert')\n",article);
								statusWindow->setText(status);
								skip=TRUE;
							}
						}
						word2=NULL;
					}
				}
			}
		}
		else if(strcmp(word1,"DATE:")==0)
		{
			strncpy(mdata->date,&buffer[6],IHEADENDSHORT);
			mdata->date[IHEADENDSHORT]=0;
			*mdata->datec = '\0';
			DateHandler::read_date(&mdata->ds,mdata->date,mdata->c_date,mdata->c_time);
		}
		else if(strcmp(word1,"MESSAGE-ID:")==0)
		{
			strncpy(mdata->messageID,&buffer[12],IHEADENDMID);
			mdata->messageID[IHEADENDMID]=0;
			char **hist = mIDhist;
			for(int l=0;l<histlen;l++) {
				if(strcmp(*hist++,mdata->messageID)==0) {
					// already downloaded!
					nLog("*** Duplicate Message! - %s (%d, %d)\n",mdata->messageID,article,l);
					sprintf(status,"Message %d Killed! (Duplicate Message)\n",article);
					statusWindow->setText(status);
					skip=TRUE;
					break;
				}
			}
		}
		else if(strcmp(word1,"CONTENT-TYPE:")==0)
		{
			char word2[256] = "";
			word(word2,buffer,2);
			if(*word2 != NULL) {
				strncpy(mdata->type,word2,IHEADENDSHORT-4);
				mdata->type[IHEADENDSHORT-4]=0;
				StripChar(mdata->type,';');
			}
		}
		else if(strcmp(word1,"REFERENCES:")==0)
		{
			char *lastref = wordLast(buffer);
			strncpy(mdata->lastref,lastref,IHEADENDMID);
			mdata->lastref[IHEADENDMID]=0;
			on_refs = TRUE;
		}
		else if(strcmp(word1,"LINES:")==0)
		{
			char word2[256] = "";
			word(word2,buffer,2);
			if(*word2 != NULL) {
				int tline = atoi(word2);
				mdata->flags[8] = tline;
				// 'lines' filter
				if(gdata->flags[2]>0) {
					if(tline>gdata->flags[2]) {
						nLog("*** Message Killed! - %d longer than %d lines (%d)\n",tline,gdata->flags[2],article);
						sprintf(status,"Message %d Killed! (Too Many Lines)\n",article);
						statusWindow->setText(status);
						skip=TRUE;
					}
				}
			}
		}
	}
	// no more checks; they must go before LINES
	// no call to doOnlineBlocking() !
	nLog("  returning %d\n",skip);
	return skip;
}

/* Hang up the connection.
 */
void hangUp(Connection *c) {
	char buffer[64]="";
	sprintf(buffer,"quit%s",newline);
	c->send_data(buffer);
	c->close();
}

/* Log into the newsserver.
 */
BOOL logIn(Connection *c,Server * server,StatusWindow * sw) {
	nLog("logIn((Connection *)%d,(Server *)%d,(StatusWindow *)%d) called\n",c,server,sw);
	char buffer[MAXLINE]="";
	char temp[1024]="";
	if(!c->call_socket(server->nntp,server->port)) {
		MUI_RequestA(app,0,0,"Error!","_Okay","\33cCannot connect with Newsserver!",0);
		return FALSE;
	}
	c->read_data(buffer);
	StripNewLine(buffer);
	if(*buffer!='2') {
		sprintf(temp,"\33cCannot use this Newsserver\nError: %s",buffer);
		MUI_RequestA(app,0,0,"Error!","_Okay",temp,0);
		hangUp(c);
		return FALSE;
	}
	//authenticate
	if(!auth(c,server,buffer,sw)) {
		sprintf(temp,"\33cCannot Log Into Newsserver!\nError: %s",buffer);
		MUI_RequestA(app,0,0,"Error!","_Okay",temp,0);
		hangUp(c);
		return FALSE;
	}
	//mode reader
	sprintf(buffer,"mode reader%s",newline);
	c->send_data(buffer);
	c->read_data(buffer);
	StripNewLine(buffer);
	if(*buffer!='2' && *buffer!='5') {
		nLog("Warning: %s received from 'mode reader' command\n",buffer);
	}
	return TRUE;
}

/* Downloads the body of a message.
 */
BOOL download_body(BOOL *available,Connection *c,BPTR file,int *bread,int *bout,StatusWindow *statusWindow) {
	nLog("download_body() called\n");
	char buffer[MAXLINE+1] = "";
	char *bufptr = NULL;
	BOOL done = FALSE;
	BOOL res = TRUE;
	int br = 0;
	int bm = 0;
	int len = c->read_data(buffer,MAXLINE);
	if(len > 0) {
		br += len;
		if(*buffer!='2') {
			*available = FALSE;
			nLog("not available!\n");
			res = FALSE;
		}
		else {
   	   bufptr = strstr(buffer,"\r\n") + 2;
	   	bm += (int)bufptr - (int)buffer;
	   }
		if(res) {
			int l = 0;
			char line[MAXLINE+1] = "";
			for(;;) {
				do_input();
				if(statusWindow->isAborted()==TRUE || running==FALSE) {
					res = FALSE;
					break;
				}
				while( bufptr < &buffer[len] ) {
					if (*bufptr != '\r')
						line[l++] = *bufptr;
					else
						bm++;
					if(l==MAXLINE-1) {
						line[l] = '\0';
						l = 0;
						if(FPuts(file,line) == -1) {
							res = FALSE;
							break;
						}
					}
					if(*bufptr++ != '\n')
						continue;
					line[l] = '\0';
					l = 0;
					if(line[0] == '.') {
						if(line[1] == '\n') {
							bm += 2;
							done = TRUE;
							break;
						}
						else {
							l = 1;
							bm++;
						}
					}
					if(FPuts(file,&line[l]) == -1) {
						res = FALSE;
						break;
					}
					l = 0;
				}
				if(done)
					break;
				len = c->read_data(buffer,MAXLINE);
				if(len<=0)
					break;
				br += len;
				bufptr = buffer;
			}
		}
	}
	else {
		nLog("no response from server!\n");
		res = FALSE;
	}
	Flush(file);
	*bread = br;
	*bout = br - bm;
	return res;
}

/* Fetch news. If 'thisgroup' equals -1, then all (subscribed) groups will
 * be checked for new messages. Otherwise, it specifies the position in the
 * group list of the single group to download from.
 */
void getnews(int thisgroup) {
	nLog("getnews((int)%d) called\n",thisgroup);
	closeSockets();
	Connection *c = new Connection();
	if(!c->init()) {
		MUI_RequestA(app,0,0,"Error!","_Okay","\33cPlease start a\nTCP stack!",0);
		delete c;
		return;
	}
	const int bufsize = 8192; // must be >= 1024 !!!
	char buffer[bufsize+1] = "";
	const int hdsize = 32768;
	char hdbuffer[hdsize+1] = "";
	char temp[256] = "";
	char status[256] = "";
	char status2[256] = "";
	static char windowtitle[256] = "";
		*windowtitle = '\0';
	char filename[300] = "";
	int time1=0,time2=0,bytes_downloaded=0;

	int entries = 0;
	getGdataEntries(&entries);
	GroupData **gdata_list = new GroupData *[entries];
	//pre-scan
	int dg=0,i=0;
	for(dg=0;dg<entries;dg++) {
		getGdata(dg,&gdata_list[dg]);
		GroupData *gdata = gdata_list[dg];
		if(gdata==NULL)
			continue;
		if( (gdata->s && thisgroup==-1) || dg==thisgroup) {
			//set correct article pointer
			if(gdata->flags[3]==0) {
				//first time
				gdata->flags[4]=0;
				if(gdata->max_dl>100 || gdata->max_dl==-1) {
					LONG result = 0;
					if(gdata->max_dl==-1) {
						sprintf(status,"\33cYou have not yet downloaded news for\n%s\nDo you wish to fetch all available messages,\nor only the last 100 messages?",gdata->name);
						result = MUI_RequestA(app,0,0,"Download","_Last 100 Messages|_All Messages",status,0);
					}
					else {
						sprintf(status,"\33cYou have not yet downloaded news for\n%s\nDo you wish to fetch the last %d messages,\nor only the last 100 messages?",gdata->name,gdata->max_dl);
						sprintf(buffer,"_Last 100 Messages|L_ast %d Messages",gdata->max_dl);
						result = MUI_RequestA(app,0,0,"Download",buffer,status,0);
					}
					if(result==0)
						gdata->flags[4]=0;
					else
						gdata->flags[4]=-1;
				}
				gdata->flags[3]=-1;
			}
		}
	}
	
	mIDhist = NULL;
	delete_dis = TRUE;
	StatusWindow statusWindow(app,"Downloading..");
	statusWindow.setText("\n");
	statusWindow.resize();
	Server *server = NULL;

	// killfiles
	nkills = 0;
	get(NLIST_acc_KILLFILE,MUIA_NList_Entries,&nkills);
	nLog("found %d killfiles\n",nkills);
	killfiles = new KillFile *[nkills];
	replace_killfile = new BOOL[nkills];
	for(i=0;i<nkills;i++) {
		KillFile *kill = NULL;
		DoMethod(NLIST_acc_KILLFILE,MUIM_NList_GetEntry,i,&kill);
		killfiles[i] = new KillFile();
		nLog("  about to copy across killfile %d\n",i);
		*killfiles[i] = *kill;
		replace_killfile[i] = FALSE;
	}

	for(dg=0;dg<entries;dg++) {
		GroupData *gdata = gdata_list[dg];
		if(gdata==NULL)
			continue;

		if( (gdata->s && thisgroup==-1) || dg==thisgroup) {

			sprintf(windowtitle,"Downloading: %s",gdata->name);
			nLog("Downloading: %s",gdata->name);
			statusWindow.setTitle(windowtitle);

			if(server==NULL || (server->ID != gdata->serverID && (gdata->serverID!=0 || server->def==FALSE))) {
				if(c->connected())
					hangUp(c);
				server = getServer(gdata->serverID);
				// open a socket
				sprintf(status,"Connecting with '%s'...\n",server->nntp);
				statusWindow.setText(status);
				if(!logIn(c,server,&statusWindow))
					break;
			}
			gdata->flags[0]=TRUE;
			//read this group
			sprintf(status,"Scanning %s\n",gdata->name);
			statusWindow.setText(status);

			sprintf(buffer,"group %s%s",gdata->name,newline);
			c->send_data(buffer);
			c->read_data(buffer);
			StripNewLine(buffer);

			if(statusWindow.isAborted()==TRUE || running==FALSE)
				break;
			if(*buffer!='2')	 {
				sprintf(temp,"\33cCannot get Group information from this Newsserver\nError: %s",buffer);
				MUI_RequestA(app,0,0,"Error!","_Okay",temp,0);
				continue;
			}

			int nidz=0;
			char cps[32]="";
			char word1[256] = "";
			char word2[256] = "";
			word(word1,buffer,3);
			word(word2,buffer,4);
			int first = atoi(word1);
			int last = atoi(word2);
			int getlast = 100;
			if(gdata->max_dl!=-1) {
				if( (gdata->max_dl>100 && gdata->flags[4]==0) || gdata->max_dl<=100)
					getlast=gdata->max_dl;
			}
			if(gdata->flags[4]==-1)
				gdata->flags[4]=last-getlast+1;
			if(gdata->flags[4]>last+1) // if flags[4]=article+1, then no new news!
				gdata->flags[4]=first;  // this test shouldn't really be needed!
			if(gdata->max_dl!=-1) {
				if(gdata->flags[4]<last-gdata->max_dl+1)
					gdata->flags[4]=last-gdata->max_dl+1;
			}
			if(gdata->flags[4]<first)
				gdata->flags[4]=first;
			if(gdata->flags[4]==last+1) {
				sprintf(status,"No new news for %s!\n",gdata->name);
				statusWindow.setText(status);
			}
			else {
				sprintf(status,"Downloading Articles: %d to %d\n",gdata->flags[4],last);
				statusWindow.setText(status);
			}
			int start = gdata->flags[4];
			if(gdata->flags[4] <= 0)
				gdata->moreflags[3]=FALSE; // need to check for message IDs
			// download first to last (inclusive)
			int bread=0,bout=0;
			int tline=0;
			int mcachepos = 0;
			if(start!=last+1) { // only if new news!
				mIDhist = NULL;
				histlen = 0;
				// only read index if we need to check for message IDs !
				nLog("up to date with newsgroup? %d\n",gdata->moreflags[3]);
				BOOL needIDs = FALSE;
				if(gdata->moreflags[3] == FALSE)
					needIDs = TRUE;
				// we now switch to false, in case there is a crash / connectionloss /
				// whatever whilst downloading - that way, next time we will check for
				// duplicates as we should.
				// otherwise, the flag will be set to TRUE (if appropriate) after
				// downloading for this group.
				gdata->moreflags[3] = FALSE;
				account.save_data(); // so this setting is saved!

				// switch to current group
				if(needIDs) {
					sprintf(status,"Reading in messageID history...\n");
					statusWindow.setText(status);
				}
				if(account.vgroupdl!=0) // change group always
					set_and_read(gdata,TRUE);

				if(needIDs) {
					Vector * vector = NULL;
					GroupData *gdata2 = NULL;
					getGdataDisplayed(&gdata2);
					// read in ID history
					if(gdata->ID!=gdata2->ID) {
						vector = new Vector(2048);
						BOOL res = read_index(gdata,vector);
						histlen=vector->getSize();
					}
					else
						get(NLIST_messagelistdata,MUIA_NList_Entries,&histlen);
					if(histlen>0) {
						mIDhist = new char *[histlen];
						nLog("about to copy messageIDs..\n");
						for(int k=0;k<histlen;k++) {
							MessageListData * mdata = NULL;
							if(vector!=NULL)
								mdata=((MessageListData **)vector->getData())[k];
							else
								DoMethod(NLIST_messagelistdata,MUIM_NList_GetEntry,k,&mdata);
							mIDhist[k] = new char[256];
							strncpy(mIDhist[k],mdata->messageID,255);
							mIDhist[k][255] = '\0';
							//mIDhist[k] = mdata->mIDhash;
						}
					}
					else
						mIDhist = NULL;
					if(vector!=NULL) {
						nLog("about to delete vector contents..\n");
						for(i=0;i<vector->getSize();i++)
							delete vector->getData()[i];
						nLog("  done\n");
						delete vector;
					}
				}
				//fetch required articles
				nidz=last-start+1;
				mcachepos=0;
				sprintf(status,"Downloading (1/%d) %s\n",nidz,cps);
				statusWindow.setText(status);
				for(int k=start;k<=last;k++) {
					nLog("  downloading %d\n",k);
					int k1 = k;
					do_input();
					if(statusWindow.isAborted()==TRUE || running==FALSE)
						break;
					sprintf(buffer,"HEAD %d%s",k1,newline);
					c->send_data(buffer);
					bytes_downloaded=0;
					time1=clock();
					int thislen = c->read_data(buffer);
					stripEscapes(buffer);
					bytes_downloaded += thislen;
					if(*buffer!='2') {
						nLog("  no head!\n");
						gdata->flags[4] = k1+1;
						k1 = -1;
					}
					int id1 = 0;
					*filename=0;
					MessageListData * mdata = NULL;
					BPTR fileout = NULL;
					if(k1!=-1) {
						for(;;) { // we want a file that doesn't exist!
							sprintf(filename,"NewsCoasterData:Folder_%d/news_%d",gdata->ID,gdata->nextmID++);
							if(!exists(filename)) {
								fileout=Open(filename,MODE_NEWFILE);
								if(fileout!=NULL)
									break;
								else
									gdata->nextmID+=10;
							}
							else
								gdata->nextmID+=10;
						}
						id1=gdata->nextmID-1;
						nLog("got file ID %d\n",id1);
					}

					if(fileout==NULL)
						k1=-1;

					if(k1!=-1) {
						mdata = new MessageListData();
						if(mdata==NULL)  {
							printf("Error - Can't Allocate mdata Object!!!\n");
							nLog("Error - Can't Allocate mdata Object!!!\n");
							delete_dis=FALSE;
							delete c;
							for(i=0;i<nkills;i++)
								delete killfiles[i];
							delete [] killfiles;
							killfiles = NULL;
							nkills = 0;
							return;
						}
						mdata->init();
						for(i=0;i<NMFLAGS;i++)
							mdata->flags[i] = 0;
						mdata->ID=id1;
						mdata->flags[6]=gdata->ID;
						mdata->size=0;
						mdata->flags[1]=TRUE;
						if(gdata->flags[5]==TRUE)
							mdata->flags[12]=TRUE;
						else
							mdata->flags[12]=FALSE;
						mdata->flags[11] = k;
					}	
					if(k1!=-1) {
						strncpy(hdbuffer,buffer,hdsize);
						hdbuffer[hdsize] = '\0';
						char *hdptr = &hdbuffer[thislen];
						if(thislen < 5 || strncmp(&buffer[thislen-5],"\r\n.\r\n",5)!=0) {
							bread = c->read_data_chunk(hdptr,hdsize - thislen);
							stripEscapes(hdptr);
							bytes_downloaded += bread;
						}
						else {
							buffer[thislen-3] = 0;
							bread = -3;
						}
						char *buffer1_2 = strstr(hdbuffer,"\r\n") +2;
						bread -= (int)buffer1_2 - (int)hdbuffer;
						Write(fileout,buffer1_2,thislen + bread);
						mdata->size += thislen + bread;

						nLog("about to process header\n");
						on_refs = FALSE;
						for(;;) {
							do_input();
							if(statusWindow.isAborted()==TRUE || running==FALSE) {
								k1 = -1;
								break;
							}
							char *buffer1_3 = strstr(buffer1_2,newline);
							if(buffer1_3 != NULL) {
								*buffer1_3 = '\0';
								nLog("  buffer1_2 = %d\n",buffer1_2);
								nLog("  buffer1_3 = %d\n",buffer1_3);
								if(processHeader(buffer1_2,gdata,mdata,gdata->name,&statusWindow,k1 - start + 1)) {
									k1=-1;
									break;
								}
								buffer1_2 = &buffer1_3[2];
								if(*buffer1_2 == '\0')
									break;
							}
							else {
								hdbuffer[bread] = '\0';
								nLog("  buffer1_2 = %d\n",buffer1_2);
								nLog("  hdbuffer  = %d\n",hdbuffer);
								nLog("  bread     = %d\n",bread);
								if(processHeader(buffer1_2,gdata,mdata,gdata->name,&statusWindow,k1 - start + 1)) {
									k1=-1;
									break;
								}
								break;
							}
						}
					}
					if(gdata->flags[5]==TRUE && k1!=-1) {
						if((account.flags & Account::QUIETDL) != 0)
							sprintf(status,"Downloading (%d/%d) %s\n",k1-start+1,nidz,cps);
						else
							sprintf(status,"Downloading (%d/%d) %s\n%s - %s",k1-start+1,nidz,cps,mdata->from,mdata->subject);
						statusWindow.setText(status);

						Write(fileout,onlineflag,onlineflag_len);
						mdata->size += onlineflag_len;
					}

					if(gdata->flags[5]==FALSE) {
						if(k1!=-1) {
							Write(fileout,"\n",1);
							mdata->size ++;
							sprintf(buffer,"BODY %d%s",k1,newline);
							c->send_data(buffer);

							if((account.flags & Account::QUIETDL) != 0)
								sprintf(status,"Downloading (%d/%d) %s\n",k1-start+1,nidz,cps);
							else
								sprintf(status,"Downloading (%d/%d) %s\n%s - %s",k1-start+1,nidz,cps,mdata->from,mdata->subject);
							statusWindow.setText(status);
						}
						if(k1!=-1) {
							// now read the body data
							BOOL available = TRUE;
							if(download_body(&available,c,fileout,&bread,&bout,&statusWindow) == FALSE) {
								if(!available)
									statusWindow.setText("BODY not found\n");
								k1 = -1;
							}
							bytes_downloaded += bread;
							mdata->size += bout;
						}
					}
					Close(fileout);
					fileout=NULL;

					if(k1==-1 || statusWindow.isAborted()==TRUE || running==FALSE) {
						nLog("  deleting file %s\n",filename);
						if(*filename!=0)
							DeleteFile(filename);
						if(mdata) {
							delete mdata;
							mdata=NULL;
						}
						nLog("  done\n");
					}
					else {
						if(*mdata->type==0)
							strcpy(mdata->type,"text/plain");
						/*GroupData * cdg = NULL;
						getGdataDisplayed(&cdg);
						if(cdg->ID==gdata->ID) {
							DoMethod(NLIST_messagelistdata,MUIM_NList_InsertSingle,mdata,MUIV_NList_Insert_Bottom);
							setEnabled();
						}*/
						write_index_single(gdata,mdata);
						gdata->nummess++;
						gdata->num_unread++;
					}

					time2 = clock() - time1;
					if(time2 == 0)
						time2++;
					sprintf(cps,"(%d cps)", (50 * bytes_downloaded) / time2 );
					if(statusWindow.isAborted()==TRUE || running==FALSE)
						break;
					gdata->flags[4]=k+1;
				}
				// end this group
			}
			DateStamp(&gdata->lastdlds);
			redrawGdataPos(dg);
			if( gdata->flags[4] > 0 && gdata->flags[4] > last ) {
				// we're up to date with the newsgroup
				// so no need to check for duplicates anymore, until we
				// reset the group article pointer
				gdata->moreflags[3] = TRUE;
			}
			account.save_data(); // save the changed group information!
			clearThreadView();
			threadView();

			if(mIDhist) {
				for(int k=0;k<histlen;k++) {
					if(mIDhist[k]) {
						delete [] mIDhist[k];
						mIDhist[k]=NULL;
					}
				}
				delete [] mIDhist;
				mIDhist=NULL;
			}
			if(statusWindow.isAborted()==TRUE || running==FALSE) {
				break;
			}
			// next newsgroup
		}
	}
	
	int enkills = 0;
	get(NLIST_acc_KILLFILE,MUIA_NList_Entries,&enkills);
	if(enkills > nkills)
		enkills = nkills;
	for(i=0;i<enkills;i++) {
		if(!replace_killfile[i])
			continue;
		KillFile *kill = killfiles[i];
		KillFile *ekill = NULL;
		DoMethod(NLIST_acc_KILLFILE,MUIM_NList_GetEntry,i,&ekill);
		// check match in case changed
		if(equals(kill->header,ekill->header) &&
			equals(kill->text,ekill->text) &&
			equals(kill->ngroups,ekill->ngroups)) {
			nLog("replacing killfile at position %d ( %s %s %s )\n",i,kill->header,kill->text,kill->ngroups);
			*ekill = *kill;
			DoMethod(NLIST_acc_KILLFILE,MUIM_NList_Redraw,i);
		}
	}
	for(i=0;i<nkills;i++)
		delete killfiles[i];
	delete [] killfiles;
	killfiles = NULL;
	delete [] replace_killfile;
	replace_killfile = NULL;
	nkills = 0;

	delete [] gdata_list;
	gdata_list = NULL;

	if(c->connected())
		hangUp(c);
	//finished all
	statusWindow.setVisible(FALSE);
	delete_dis=FALSE;

	delete c;
	return;
}

BOOL getBody(BOOL *available,GroupData * gdata,MessageListData * mdata) {
	StatusWindow * statusWindow = new StatusWindow(app,"Downloading body..");
	BOOL res = getBody(available,gdata,mdata,statusWindow,-1,-1,TRUE,TRUE);
	delete statusWindow;
	return res;
}

BOOL getBody(BOOL *available,GroupData * gdata,MessageListData * mdata,StatusWindow * statusWindow,int index,int max,BOOL first, BOOL delmess) {
	nLog("getBody((GroupData *)%d,(MessageListData *)%d,(StatusWindow *)%d,(int)%d,(int)%d,(BOOL)%d,(BOOL)%d) called\n",gdata,mdata,statusWindow,index,max,first,delmess);
	if(online_c == NULL) {
		online_c = new Connection();
		if(!online_c->init()) {
			MUI_RequestA(app,0,0,"Error!","_Okay","\33cPlease start a\nTCP stack!",0);
			delete online_c;
			online_c = NULL;
			return FALSE;
		}
	}
	*available = TRUE;
	//ViewWindow::sleepAll(TRUE);
	/*statusWindow->setText("\n");
	statusWindow->resize();*/
	Server *server = getServer(gdata->serverID);
	GroupData * gdata_curr=NULL;
	BOOL OKAY = TRUE;
	const int bufsize = 4096; // must be >= 1024 !!!
	char buffer[bufsize+1] = "";
	char temp[256] = "";
	static char textbuf[256] = "";
	int bytes_downloaded=0;
	int bread = 0,bout = 0;

	int entries = 0;
	getGdataEntries(&entries);
	
	// switch to current group
	set_and_read(gdata,TRUE);
	bytes_downloaded=0;
	delete_dis=TRUE;

	char filename[300] = "";
	getFilePath(filename,gdata->ID,mdata->ID);
	BPTR lock = Lock(filename,ACCESS_READ); // READ?
	BPTR fileout = Open(filename,MODE_OLDFILE);

	LONG oldpos = 0;
	int oldsize = mdata->size;
	BOOL xheader_zapped = FALSE;
	/*if(fileout!=NULL && lock!=NULL) {
		Seek(fileout,-onlineflag_len,OFFSET_END);
		Read(fileout,buffer,onlineflag_len);
		if(strncmp(buffer,onlineflag,onlineflag_len)==0) {
			Seek(fileout,-onlineflag_len,OFFSET_CURRENT);
			mdata->size -= onlineflag_len;
		}
		else
			Seek(fileout,0,OFFSET_END);
		oldpos = Seek(fileout,0,OFFSET_CURRENT);
		mdata->flags[12]=FALSE;
		gdata->flags[1]=TRUE;
		Write(fileout,"\n",strlen("\n"));
		mdata->size++;
	}*/
	
	char bodyline[MAXLINE] = "";
	if(strlen(mdata->messageID) >= IHEADENDMID-1) {
		NewMessage nm;
		strcpy(nm.getThisHeader,"Message-ID:");
		get_refs(&nm,gdata->ID,mdata->ID,GETREFS_NONE);
		sprintf(bodyline,"BODY %s%s",nm.dummyHeader,newline);
	}
	else
		sprintf(bodyline,"BODY %s%s",mdata->messageID,newline);

	set(menuitem_DISCONNECT,MUIA_Menuitem_Enabled,FALSE);
	
	BOOL connected = FALSE;
	
	//while(!connected) {
	for(int attempts=0;attempts<2 && !connected;attempts++) {
		nLog("attempt number %d\n",attempts);
	
		connected = TRUE;

		if(!online_c->connected()) {
			sprintf(textbuf,"Connecting with '%s'...\n",server->nntp);
			statusWindow->setText(textbuf);
			if(!online_c->call_socket(server->nntp,server->port)) {
				MUI_RequestA(app,0,0,"Error!","_Okay","\33cCannot connect with Newsserver!",0);
				OKAY = FALSE;
			}
			if(OKAY) {
				online_c->read_data(buffer);
				StripNewLine(buffer);
				if(*buffer!='2' || statusWindow->isAborted()==TRUE) {
					if(statusWindow->isAborted()==FALSE) {
						sprintf(temp,"\33cCannot use this Newsserver\nError: %s",buffer);
						MUI_RequestA(app,0,0,"Error!","_Okay",temp,0);
					}
					sprintf(buffer,"quit%s",newline);
					online_c->send_data(buffer);
					OKAY = FALSE;
				}
			}
			//authenticate
			if(OKAY && !auth(online_c,server,buffer,statusWindow)) {
				sprintf(temp,"\33cCannot Log Into Newsserver!\nError: %s",buffer);
				MUI_RequestA(app,0,0,"Error!","_Okay",temp,0);
				sprintf(buffer,"quit%s",newline);
				online_c->send_data(buffer);
				OKAY = FALSE;
			}
			if(OKAY) {
				sprintf(buffer,"mode reader%s",newline);
				online_c->send_data(buffer);
				online_c->read_data(buffer);
				StripNewLine(buffer);
				if(statusWindow->isAborted()==FALSE) {
					if(*buffer!='2' && *buffer!='5') {
						nLog("Warning: %s received from 'mode reader' command\n",buffer);
					}
				}
			}
		}
		if(!OKAY) {
			nLog("not okay - exiting loop\n");
			break;
		}
	
		if(index > 0 && max > 0) {
			sprintf(textbuf,"Downloading %d of %d..\n",index,max);
			statusWindow->setText(textbuf);
		}
		else
			statusWindow->setText("Downloading...\n");

		statusWindow->resize();

		if(first) {
			//printf("group %s\n",gdata->name);
			sprintf(buffer,"group %s%s",gdata->name,newline);
			online_c->send_data(buffer);
			online_c->read_data(buffer);
		}

		if(fileout!=NULL && lock!=NULL) {

			if(!xheader_zapped) {
				xheader_zapped = TRUE;
				Seek(fileout,-onlineflag_len,OFFSET_END);
				Read(fileout,buffer,onlineflag_len);
				if(strncmp(buffer,onlineflag,onlineflag_len)==0) {
					Seek(fileout,-onlineflag_len,OFFSET_CURRENT);
					mdata->size -= onlineflag_len;
				}
				else
					Seek(fileout,0,OFFSET_END);
				oldpos = Seek(fileout,0,OFFSET_CURRENT);
				// only set online flag afterwards, if necessary
				/*mdata->flags[12]=FALSE;
				gdata->flags[1]=TRUE;*/
				Write(fileout,"\n",1);
				mdata->size++;
			}

			for(int go=0;go<2;go++) {
				nLog("(%d) sending: %s\n",go,bodyline);
				online_c->send_data(bodyline);
				// now read the body data
				if(download_body(available,online_c,fileout,&bread,&bout,statusWindow)) {
					// ok!
					nLog("ok!\n");
					break;
				}
				else {
					nLog("failed - available = %d\n",*available);
					if(bread <= 0) {
						nLog("lost connection?\n");
						connected = FALSE; // lost connection?
						first = TRUE;
						online_c->close();
						break;
					}
					else {
						BOOL another_go = TRUE;
						if(go==0) {
							// try with article number?
							int id = mdata->flags[11];
							nLog("trying with article number %d\n",id);
							if(id==0)
								another_go = FALSE;
							if( *available )
								another_go = FALSE; // don't try again if the problem was something other than BODY not available
							if(another_go) {
								sprintf(buffer,"group %s%s",gdata->name,newline);
								nLog("send: %s\n",buffer);
								online_c->send_data(buffer);
								online_c->read_data(buffer);
								if(*buffer != '2') {
									nLog("group command returned %s - fail\n",buffer);
									another_go = FALSE;
								}
							}
							if(another_go) {
								*available = TRUE; // reset!
								sprintf(buffer,"stat %d%s",id,newline);
								online_c->send_data(buffer);
								nLog("send: %s\n",buffer);
								online_c->read_data(buffer);
								nLog("rcvd: %s\n",buffer);
								if(*buffer != '2')
									another_go = FALSE;
								else {
									char word3[1024] = "";
									word(word3,buffer,3);
									if(stricmp(word3,mdata->messageID)!=0)
										another_go = FALSE;
								}
								if(!another_go)
									*available = FALSE; // put back again
							}
							if(another_go) {
								sprintf(bodyline,"body %d%s",id,newline);
							}
						}
						if(go==1 || !another_go) {
							if( !(*available) )
								statusWindow->setText("This message is no longer available!\n");
							OKAY=FALSE;
							break;
						}
					}
				}
			}
		}
	}
	if(!connected) {
		// couldn't get connection
		OKAY = FALSE;
	}
	if(fileout!=NULL && lock!=NULL) {
		bytes_downloaded += bread;
		mdata->size += bout;
		if( statusWindow->isAborted()==TRUE || running==FALSE || OKAY==FALSE ) {
			if( xheader_zapped ) {
				Flush(fileout);
				Seek(fileout,oldpos,OFFSET_BEGINNING);
				Write(fileout,onlineflag,onlineflag_len);
				mdata->size = oldpos + onlineflag_len;
				// flags[12] should still be TRUE
				//mdata->flags[12]=TRUE;
			}
		}
		else {
			mdata->flags[12]=FALSE;
			gdata->flags[1]=TRUE;
			if(mdata->size < oldsize) {
				// small bodies - we need to overwrite the 'onlineflag' text
				nLog("small body case\n");
				if(SetFileSize(fileout,mdata->size,OFFSET_BEGINNING) == -1) {
					nLog("SetFileSize() failed - pad out with extra bytes\n");
					// Alternative to shrinking the file size
					for(int i=0;i<oldsize - mdata->size;i++)
						FPutC(fileout,'\0');
					mdata->size = oldsize;
				}
			}
		}
		Close(fileout);
		fileout=NULL;
		UnLock(lock);
		lock=NULL;
	}
	else {
		//printf("Can't Write To File: %s\n",filename);
		nLog("Can't Write To File: %s\n",filename);
		OKAY=FALSE;
		sprintf(buffer,"\33cUnable to open file:\n%s",filename);
		MUI_RequestA(app,0,0,"Error","_Okay",buffer,0);
	}
	if(max == -1)
		redrawGdataAll();
	
	//finished all
	delete_dis=FALSE;
	if(statusWindow->isAborted()==TRUE || running==FALSE || OKAY==FALSE) {
		closeSockets();
	}
	else {
		set(menuitem_DISCONNECT,MUIA_Menuitem_Enabled,TRUE);
		online_c->doOnlineBlocking(FALSE,FALSE);
	}
	//ViewWindow::sleepAll(FALSE);

	nLog("  available = %d\n",*available);
	if( !(*available) ) {
		if(delmess && (account.flags & Account::NODELONLINE)==0) {
			// delete message
			nLog(" >>> deleting message header\n");
			delete_mess(gdata,mdata,FALSE,FALSE); // close==FALSE - we don't want to delete this window!
		}
	}
	return OKAY;
}

void closeSockets() {
	nLog("closeSockets() called\n");
	set(menuitem_DISCONNECT,MUIA_Menuitem_Enabled,FALSE);
	if(online_c != NULL) {
		hangUp(online_c);
		delete online_c;
		online_c = NULL;
	}
}