timeout on initial connect; basic cache scrubbing, needs work; another memory optimalisation

trunk/ripnews/news/article.rb

@@ -1,4 +1,4 @@
-# $Dwarf: article.rb,v 1.89 2004/09/01 11:25:46 ward Exp $
+# $Dwarf: article.rb,v 1.90 2004/10/14 11:46:31 ward Exp $
 # $Source$
 
 #
@@ -53,11 +53,11 @@ def initialize(nntpservers, groupname, newsrc="~/.newsrc")
 			p server
 			p Time.now
 			begin
-				timeout(180) do
+				timeout(60) do
 					@connections[server]["nntp"] = Net::NNTP.new(server)
 				end
 			rescue TimeoutError
-				puts "Timeout due to sucky server, reconnecting"
+				sleep 3
 				retry
 			end
 			p Time.now
@@ -72,6 +72,7 @@ def initialize(nntpservers, groupname, newsrc="~/.newsrc")
 end
 
 def reconnect(server)
+	retries = 0
 	begin
 		@connections[server]["nntp"].quit
 		# helpt dit in geheugen gebruik? : Volgens mij niet
@@ -81,13 +82,19 @@ def reconnect(server)
 	end
 	begin
 		sleep 3
-		timeout(180) do
+		#timeout(180) do
+		timeout(60) do
 			@connections[server]["nntp"] = Net::NNTP.new(server)
 		end
 	rescue SocketError, Errno::EINVAL, EOFError, Errno::ETIMEDOUT, TimeoutError
 		print "Reconnect to #{server} failed: #{$!}\n"
+		if retries > 1
 			del_server(server)
 			raise PermError, "Couldn't connect to #{server}"
+		else
+			retries += 1
+			retry
+		end
 	end
 	print "Succesfully reconnected to #{server}\n"
 end
@@ -158,8 +165,8 @@ def get_articles(cachedir=false)
 				print "	Last: #{last}\n"
 			end
 			# clean up old newsrc entries
-			if @connections[server]["first"].to_i > 0
+			if @connections[server]["first"] > 0
-				@connections[server]["newsrc"].unmark_range(@group, 0, (@connections[server]["first"].to_i-1).to_s)
+				@connections[server]["newsrc"].unmark_range(@group, 0, (@connections[server]["first"] - 1).to_s)
 				@connections[server]["newsrc"].save
 			end
 		else
@@ -562,14 +569,14 @@ p Time.now
 		@connections[server]["skip_ids"].elements.collect!{|x| excludes[server][x]=true}
 		if FileTest.directory?( cachedir) and FileTest.file?( "#{filename}.#{server}" ) and FileTest.readable?( "#{filename}.#{server}" )
 			file = File.new( "#{filename}.#{server}" )
-			lines = file.readlines
+			file.each{|line|
-			lines.collect{|line|
 				# id | messageid | subject
 				#if line =~ /^(\d+)\|(.*?)\|(.*)$/
-				if lineregexp.match(line) != nil
+				if line =~ lineregexp
+				#if lineregexp.match(line) != nil
 					unless excludes.has_key?(server) and excludes[server].has_key?($1.to_i) or
-						$1.to_i < @connections[server]["first"].to_i or
+						$1.to_i < @connections[server]["first"] or
-						$1.to_i > @connections[server]["last"].to_i
+						$1.to_i > @connections[server]["last"]
 						if preselect($3)
 							add($2, $1, server, $3)
 						end
@@ -609,6 +616,32 @@ p Time.now
 			print "Couldn't update #{server} cache\n"
 		end
 	end
+p Time.now
+	scrub_cache(cachedir, server)
+end
+
+def scrub_cache(cachedir, server)
+# XXX this could and probably should be done in a separate thread...
+# XXX but it'll work for now
+p "scrubbing cache"
+p Time.now
+	filename = "#{cachedir}/#{@group}.ripnewscache"
+	regexp = Regexp.new('^(\d+)\|')
+	infile = File.new("#{filename}.#{server}") or puts "Couldn't open cachefile for reading"
+	outfile = File.new("#{filename}.#{server}.new", "w") or puts "Couldn't open cachefile for writing"
+	infile.each{ |line|
+		if line =~ regexp
+			if $1.to_i >= @connections[server]["first"] and
+			   $1.to_i <= @connections[server]["last"]
+				outfile.puts(line)
+			end
+		end
+	}
+	#if ( File.move("#{filename}.#{server}.new", "#{filename}.#{server}") )
+	#	print "Cache scrubbed for #{server}\n"
+	#else
+	#	print "Couldn't scrub #{server} cache\n"
+	#end
 p Time.now
 end