From 760c879fc4bd19726a501bc051bf827c3a40732f Mon Sep 17 00:00:00 2001
From: Ward Wouts <ward@wouts.nl>
Date: Sun, 6 Mar 2005 11:14:28 +0000
Subject: [PATCH] a bit faster cache reading

---
 trunk/ripnews/news/article.rb | 59 +++++++++++++++++------------------
 1 file changed, 29 insertions(+), 30 deletions(-)

diff --git a/trunk/ripnews/news/article.rb b/trunk/ripnews/news/article.rb
index 7ba090b..8ee6300 100644
--- a/trunk/ripnews/news/article.rb
+++ b/trunk/ripnews/news/article.rb
@@ -1,4 +1,4 @@
-# $Dwarf: article.rb,v 1.110 2005/03/01 19:55:32 ward Exp $
+# $Dwarf: article.rb,v 1.111 2005/03/06 07:01:08 ward Exp $
 # $Source$
 
 #
@@ -112,8 +112,8 @@ def initialize(nntpservers, groupname, newsrc="~/.newsrc")
 		@connections[server] = {}
 		@cache_buf[server] = []
 		begin
-			p server
-			p Time.now
+#			p server
+#			p Time.now
 			begin
 				timeout(60) do
 					@connections[server]["nntp"] = Net::KANNTP.new(server)
@@ -123,7 +123,7 @@ def initialize(nntpservers, groupname, newsrc="~/.newsrc")
 				sleep 3
 				retry
 			end
-			p Time.now
+#			p Time.now
 			@connections[server]["skip_ids"] = Set::IntSpan.new()
 			@connections[server]["newsrc"] = News::Newsrc.new("#{newsrc}.#{server}")
 			set_skip_ids(server, @connections[server]["newsrc"].marked_articles(@group))
@@ -268,7 +268,6 @@ def get_articles(cachedir=false)
 		print "rangelist elements: #{range.diff(@connections[server]["skip_ids"]).elements}\n" if Debuglevel > 2
 		begin
 		unless rangelist == nil or rangelist =~ /^$/
-			headerlines = 0
 			for i in rangelist.split(',')
 				print "i: #{i}\n" if Debuglevel > 2
 				begin
@@ -311,12 +310,6 @@ def get_articles(cachedir=false)
 						cache_add(cachedir, id, art[id]["messid"], art[id]["subject"], server)
 					end
 				end
-#				headerlines += xover_lines.length
-				headerlines += subj_lines.length
-				if headerlines >= 500
-					cache_save(cachedir, server)
-					headerlines = 0
-				end
 			end
 		end
 		rescue PermError
@@ -720,28 +713,40 @@ def cache_check(cachedir)
 end
 
 def cache_read(cachedir)
-p "reading cache"
+p "reading & scrubbing cache"
 p Time.now
 	filename = "#{cachedir}/#{@group}.ripnewscache"
 	excludes = {}
 	for server in @connections.keys
-		cache_scrub(cachedir, server)
+		#cache_scrub(cachedir, server)
+puts "	reading cache for #{server}"
+puts "	#{Time.now}"
 		excludes[server] = {}
 		@connections[server]["skip_ids"].elements.collect!{|x| excludes[server][x]=true}
 		if FileTest.directory?( cachedir) and FileTest.file?( "#{filename}.#{server}" ) and FileTest.readable?( "#{filename}.#{server}" )
+			outfile = File.new("#{filename}.#{server}.new", "w") or puts "Couldn't open cachefile for writing"
 			File.new( "#{filename}.#{server}" ).each{ |line|
 				id, messid, subject = line.split("|", 3)
-				unless excludes.has_key?(server) and excludes[server].has_key?(id.to_i) or
-					id.to_i < @connections[server]["first"] or
-					id.to_i > @connections[server]["last"]
-					if preselect(subject)
-						add(id, messid, subject, server)
+				id_i = id.to_i
+				if id_i >= @connections[server]["first"] and
+				   id_i <= @connections[server]["last"]
+					outfile.puts(line)
+					unless excludes[server].has_key?(id_i)
+						if preselect(subject)
+							add(id_i, messid, subject, server)
+						end
+						@connections[server]["skip_ids"].insert(id_i)
 					end
-					@connections[server]["skip_ids"].insert(id.to_i)
 				end
 			}
+			if ( File.move("#{filename}.#{server}.new", "#{filename}.#{server}") )
+				print "Cache scrubbed for #{server}\n"
+			else
+				print "Couldn't scrub #{server} cache\n"
+			end
 		end
 	end
+p "caches read"
 p Time.now
 #memusage
 end
@@ -771,22 +776,16 @@ p "scrubbing cache"
 p Time.now
 	filename = "#{cachedir}/#{@group}.ripnewscache"
 	if File.exists?("#{filename}.#{server}")
-		regexp = Regexp.new('^(\d+)\|')
+#		regexp = Regexp.new('^(\d+)\|')
 		infile = File.new("#{filename}.#{server}") or puts "Couldn't open cachefile for reading"
 		outfile = File.new("#{filename}.#{server}.new", "w") or puts "Couldn't open cachefile for writing"
 		infile.each{ |line|
-			if line =~ regexp
-				if $1.to_i >= @connections[server]["first"] and
-				   $1.to_i <= @connections[server]["last"]
-					outfile.puts(line)
-				end
+			id, messid, subject = line.split("|", 3)
+			if id.to_i >= @connections[server]["first"] and
+			   id.to_i <= @connections[server]["last"]
+				outfile.puts(line)
 			end
 		}
-		if ( File.move("#{filename}.#{server}.new", "#{filename}.#{server}") )
-			print "Cache scrubbed for #{server}\n"
-		else
-			print "Couldn't scrub #{server} cache\n"
-		end
 	end
 p Time.now
 end