diff --git a/trunk/ripnews/ripnews.rb b/trunk/ripnews/ripnews.rb index a162d8e..6ccdc57 100755 --- a/trunk/ripnews/ripnews.rb +++ b/trunk/ripnews/ripnews.rb @@ -1,6 +1,6 @@ #!/usr/local/bin/ruby -w -# $Dwarf: ripnews.rb,v 1.83 2004/06/16 10:49:54 ward Exp $ +# $Dwarf: ripnews.rb,v 1.84 2004/09/01 11:24:15 ward Exp $ # $Source$ # @@ -29,9 +29,52 @@ require 'tempfile' require 'encode/uuencode' require 'encode/yenc' +########################################################################### ########################################################################### +# memory profiling stuff +MEntry = Struct.new( "MEntry", :c, :mem ) +class MEntry; def to_s() "#{c} : #{mem}"; end; end + +GroupEntry = Struct.new( "GroupEntry", :c, :mem, :total ) +class GroupEntry; def to_s() "#{mem}\t\t#{c} x#{total}"; end; end + +def profile_mem(group) +end +def aprofile_mem(group) + t = Thread.new { + groups = {} + ObjectSpace.each_object { |x| + if not [Array,Hash].include? x.class + e = nil + begin + e = MEntry.new( x.class, Marshal::dump(x).size ) + rescue TypeError # undumpable + e = MEntry.new( x.class, 0 ) + end + if groups.has_key? e.c + groups[e.c].mem += e.mem + groups[e.c].total += 1 + else + groups[e.c] = GroupEntry.new( e.c, e.mem, 1 ) + end + end + } + File.open( "mem_log", "a+" ) { |file| + file << "Group #{group}\n" + total = 0 + file << "bytes/class/count\n" + groups.to_a.sort_by { |e| e[1].mem }.each { |e| + file << "#{e[1]}\n"; total += e[1].mem } + file << "TOTAL == #{total}\n\n" + } + } + sleep 10 + t.join +end +########################################################################### + Debuglevel = 0 @tstart = Time.now @@ -402,6 +445,12 @@ def get_multi(subj, group) end end +def fill_preselector(group) + if @config[group].has_key?("-I") + @articles.add_preselect_pattern(Regexp.new(@config[group]["-I"])) + end +end + def output_data(subject, mode, filename="", body="") group = @articles.get_groupname print " mode: #{mode}\n" if Debuglevel > 0 @@ -537,11 +586,19 @@ def startup end def main + profile_mem("out side of loop still") for group in @config.keys.sort + profile_mem("#{group} start") + puts "object count:" + puts ObjectSpace.each_object(){} print "\nGetting articles for #{group}\n" @articles = Article.new(@config[group]["NNTPSERVER"], group, @config[group]["NEWSRCNAME"]) + fill_preselector(group) + print "initialized\n" @articles.get_articles(@config[group]["CACHEDIR"]) + profile_mem("#{group} articles read") + unless FileTest.directory?("#{@config[group]["DATADIR"]}/#{group}") or Dir.mkdir("#{@config[group]["DATADIR"]}/#{group}", @config[group]["PERMISSION"].oct) print "eeeps, couldn't create dir\n" @@ -582,7 +639,9 @@ def main end @articles.quit @articles = nil + profile_mem("#{group} pre-GC") GC.start + profile_mem("#{group} end") end end