memory profiling option, will probably remove this. major memory usage improvements

trunk/ripnews/ripnews.rb

@@ -1,6 +1,6 @@
 #!/usr/local/bin/ruby -w
 
-# $Dwarf: ripnews.rb,v 1.83 2004/06/16 10:49:54 ward Exp $
+# $Dwarf: ripnews.rb,v 1.84 2004/09/01 11:24:15 ward Exp $
 # $Source$
 
 #
@@ -29,9 +29,52 @@ require 'tempfile'
 require 'encode/uuencode'
 require 'encode/yenc'
 
+###########################################################################
 
 ###########################################################################
 
+# memory profiling stuff
+MEntry = Struct.new( "MEntry", :c, :mem )
+class MEntry; def to_s() "#{c} : #{mem}"; end; end
+
+GroupEntry = Struct.new( "GroupEntry", :c, :mem, :total )
+class GroupEntry; def to_s() "#{mem}\t\t#{c} x#{total}"; end; end
+
+def profile_mem(group)
+end
+def aprofile_mem(group)
+	t = Thread.new {
+		groups = {}
+		ObjectSpace.each_object { |x|
+			if not [Array,Hash].include? x.class
+			e = nil
+			begin
+				e = MEntry.new( x.class, Marshal::dump(x).size )
+			rescue TypeError # undumpable
+				e = MEntry.new( x.class, 0 )
+			end
+			if groups.has_key? e.c
+				groups[e.c].mem += e.mem
+				groups[e.c].total += 1
+			else
+				groups[e.c] = GroupEntry.new( e.c, e.mem, 1 )
+			end
+		end
+	}
+		File.open( "mem_log", "a+" ) { |file|
+			file << "Group #{group}\n"
+			total = 0
+			file << "bytes/class/count\n"
+			groups.to_a.sort_by { |e| e[1].mem }.each { |e|
+				file << "#{e[1]}\n"; total += e[1].mem }
+			file << "TOTAL == #{total}\n\n"
+		}
+	}
+	sleep 10
+	t.join
+end
+###########################################################################
+
 Debuglevel = 0
 @tstart = Time.now
 
@@ -402,6 +445,12 @@ def get_multi(subj, group)
 	end
 end
 
+def fill_preselector(group)
+	if @config[group].has_key?("-I")
+		@articles.add_preselect_pattern(Regexp.new(@config[group]["-I"]))
+	end
+end
+
 def output_data(subject, mode, filename="", body="")
 	group = @articles.get_groupname
 	print "	mode: #{mode}\n" if Debuglevel > 0
@@ -537,11 +586,19 @@ def startup
 end
 
 def main
+	profile_mem("out side of loop still")
 	for group in @config.keys.sort
+		profile_mem("#{group} start")
+		puts "object count:"
+		puts ObjectSpace.each_object(){}
 		print "\nGetting articles for #{group}\n"
 		@articles = Article.new(@config[group]["NNTPSERVER"], group, @config[group]["NEWSRCNAME"])
+		fill_preselector(group)
+		print "initialized\n"
 		@articles.get_articles(@config[group]["CACHEDIR"])
 
+		profile_mem("#{group} articles read")
+
 		unless FileTest.directory?("#{@config[group]["DATADIR"]}/#{group}") or
 				Dir.mkdir("#{@config[group]["DATADIR"]}/#{group}", @config[group]["PERMISSION"].oct)
 			print "eeeps, couldn't create dir\n"
@@ -582,7 +639,9 @@ def main
 		end
 		@articles.quit
 		@articles = nil
+		profile_mem("#{group} pre-GC")
 		GC.start
+		profile_mem("#{group} end")
 	end
 end