diff --git a/encode/uuencode.rb b/encode/uuencode.rb index 6058f65..77c4f67 100644 --- a/encode/uuencode.rb +++ b/encode/uuencode.rb @@ -90,7 +90,8 @@ def _uudecode_file(file, outfile) next if line =~ /[a-z]/ next if line == nil next unless ((((line[0].ord - 32) & 077) + 2) / 3).to_i == (line.length/4).to_i - line.unpack("u").each{|x| outfile.print x} + #line.unpack("u").each{|x| outfile.print x} + line.unpack("u").each{|x| outfile.write x} end puts "No \"end\" found!!!" diff --git a/news/article.rb b/news/article.rb index cbfe52a..8118343 100644 --- a/news/article.rb +++ b/news/article.rb @@ -1,6 +1,3 @@ -# $Dwarf: article.rb,v 1.114 2005/05/12 07:39:53 ward Exp $ -# $Source$ - # # Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 Ward Wouts # @@ -23,7 +20,7 @@ require Pathname.new(__FILE__).dirname + '../news/newsrc' require 'tempfile' require 'timeout' #require 'yaml' -require 'profiler' +#require 'profiler' # removed from ruby? class ArticleError < RuntimeError; end class TempError < ArticleError; end @@ -114,6 +111,10 @@ def initialize(nntpservers, groupname, newsrc="~/.newsrc", maxage=0) @serverlist = [] @serverpasses = {} @maxage = maxage.to_i + @oldestallowed = 0 + if @maxage != 0 + @oldestallowed = (DateTime.now - @maxage).strftime('%Y%m%d').to_i + end tmplist = nntpservers.split('|') tmplist.each{ |server| @@ -148,7 +149,7 @@ def initialize(nntpservers, groupname, newsrc="~/.newsrc", maxage=0) # p server # p Time.now begin - timeout(60) do + Timeout.timeout(60) do #p "connecting" @connections[server]["nntp"] = Net::KANNTP.new(server, 119, @serverpasses[server]['user'], @serverpasses[server]['pass']) end @@ -176,7 +177,7 @@ def reconnect(server) retries = 0 begin puts "Trying to kill old connection #{Time.now}" - timeout(10) do + Timeout.timeout(10) do @connections[server]["nntp"].quit end puts "Killed old connection #{Time.now}" @@ -188,7 +189,7 @@ def reconnect(server) puts "Trying to reconnect #{Time.now}" sleep 3 #timeout(180) do - timeout(60) do + Timeout.timeout(60) do @connections[server]["nntp"] = Net::KANNTP.new(server, 119, @serverpasses[server]['user'], @serverpasses[server]['pass']) end resp = @connections[server]["nntp"].mode_reader @@ -331,7 +332,8 @@ def get_articles(cachedir=false) art[id] = {} unless art.has_key?(id) begin - lastdate = art[id]["date"] = DateTime.parse(date).strftime('%Y%m%d') + lastdate = DateTime.parse(date) + art[id]["date"] = DateTime.parse(date).strftime('%Y%m%d') rescue puts $!.message puts id @@ -354,14 +356,16 @@ def get_articles(cachedir=false) # dit wellicht alleen doen indien preselector hem uitkiest # en anders een leuk regeltje aan de cache toevoegen, # maar niet in het geheugen houden - if preselect(art[id]["subject"]) + if art[id]["date"].to_i >= @oldestallowed && preselect(art[id]["subject"]) add(id.to_i, art[id]["messid"], art[id]["date"], art[id]["from"], art[id]["subject"], server) end - cache_add(cachedir, id, art[id]["messid"], art[id]["date"], art[id]["from"], art[id]["subject"], server) + if art[id]["date"].to_i >= @oldestallowed + cache_add(cachedir, id, art[id]["messid"], art[id]["date"], art[id]["from"], art[id]["subject"], server) + end end } if @maxage and @maxage > 0 - if DateTime.parse(lastdate) < ( DateTime.now - @maxage ) + if lastdate < ( DateTime.now - @maxage ) puts "Skipping articles older than #{DateTime.now - @maxage}" break end @@ -384,7 +388,7 @@ def get_group_info(server) first = "" last = "" begin - timeout(30) do + Timeout.timeout(30) do begin resp, count, first, last, name = @connections[server]["nntp"].group(@group) rescue Net::NNTPReplyError @@ -418,7 +422,7 @@ def get_xhdr(server, range, header) resp = "" lines = [] begin - timeout(180) do + Timeout.timeout(180) do begin p Time.now if Debuglevel > 1 puts "getting headers: #{header}, #{range}" if Debuglevel > 1 @@ -474,7 +478,7 @@ def get_xover(server, range) ed = start end begin - timeout(180) do + Timeout.timeout(180) do begin p Time.now if Debuglevel > 1 puts "getting headers: #{range}" if Debuglevel > 1 @@ -532,7 +536,7 @@ def get_body(server, message) messid = "" list = [] begin - timeout(180) do + Timeout.timeout(180) do begin list = [] resp, id, messid, list = @connections[server]["nntp"].body(message) @@ -825,6 +829,7 @@ def save_newsrc() end def cache_add(cachedir, id, messid, date, from, subject, server) + # also add to skip stuff if @cache_buf.has_key?(server) @cache_buf[server].push("#{id}|#{messid}|#{date}|#{from}|#{subject}\n") else @@ -849,7 +854,7 @@ puts "#{Time.now} Reading & scrubbing caches" @connections.keys.each{|server| first = @connections[server]["first"] last = @connections[server]["last"] - #cache_scrub(cachedir, server) + cache_scrub(cachedir, server) puts " #{Time.now} Reading cache for #{server}" excludes[server] = {} @connections[server]["skip_ids"].elements.collect!{|x| excludes[server][x]=true} @@ -876,11 +881,13 @@ puts " #{Time.now} Reading cache for #{server}" if first <= id_i and id_i <= last if ! excludes[server].has_key?(id_i) outfile.puts(line) - if preselect(subject) + if date.to_i >= @oldestallowed && preselect(subject) add(id_i, messid, date, from, subject, server) end # XXX alle traagheid van de cache_read zit in deze regel: - @connections[server]["skip_ids"].insert!(id_i) + if date.to_i < @oldestallowed + @connections[server]["skip_ids"].insert!(id_i) + end end end end @@ -935,11 +942,19 @@ p Time.now outfile = File.new("#{filename}.#{server}.new", "w") or puts "Couldn't open cachefile for writing" infile.each{ |line| id, messid, date, subject = line.split("|", 3) + #puts "#{date.to_i} #{@oldestallowed}" + # XXX maybe also add to skipids ?? + next if date.to_i < @oldestallowed if id.to_i >= @connections[server]["first"] and id.to_i <= @connections[server]["last"] outfile.puts(line) end } + if ( FileUtils.move("#{filename}.#{server}.new", "#{filename}.#{server}") ) + puts " #{Time.now} Cache scrubbed for #{server}" + else + puts "Couldn't scrub #{server} cache" + end end p Time.now end diff --git a/ripnews.rb b/ripnews.rb index 1f81e12..2af18c7 100755 --- a/ripnews.rb +++ b/ripnews.rb @@ -50,20 +50,20 @@ def aprofile_mem(group) groups = {} ObjectSpace.each_object { |x| if not [Array,Hash].include? x.class - e = nil - begin - e = MEntry.new( x.class, Marshal::dump(x).size ) - rescue TypeError # undumpable - e = MEntry.new( x.class, 0 ) + e = nil + begin + e = MEntry.new( x.class, Marshal::dump(x).size ) + rescue TypeError # undumpable + e = MEntry.new( x.class, 0 ) + end + if groups.has_key? e.c + groups[e.c].mem += e.mem + groups[e.c].total += 1 + else + groups[e.c] = GroupEntry.new( e.c, e.mem, 1 ) + end end - if groups.has_key? e.c - groups[e.c].mem += e.mem - groups[e.c].total += 1 - else - groups[e.c] = GroupEntry.new( e.c, e.mem, 1 ) - end - end - } + } File.open( "mem_log", "a+" ) { |file| file << "Group #{group}\n" total = 0 @@ -547,11 +547,17 @@ def get_multi(subj, group) # puts "inside thread post pass\n" if UUEncode.is_uuencoded(tbody) puts " UUDecoding..." - if tfile - tmode, tfilename, tbody = UUEncode.uudecode(tfile, tfileout) - else - tmode, tfilename, tbody = UUEncode.uudecode(tbody) - end + begin + if tfile + tmode, tfilename, tbody = UUEncode.uudecode(tfile, tfileout) + else + tmode, tfilename, tbody = UUEncode.uudecode(tbody) + end + rescue Encoding::UndefinedConversionError + puts "#{$!}" + puts " Skipping article..." + Thread.current.exit + end elsif YEnc.is_yencoded(tbody) puts " YDecoding..." begin @@ -683,7 +689,7 @@ def get_max_file_length(tempdir=".") name = "a"*500 name = "#$$#{name}" begin - file = File.new("#{tempdir}/#{name}", "w", 0644).close + File.new("#{tempdir}/#{name}", "w", 0644).close FileUtils.rm("#{tempdir}/#{name}") rescue Errno::ENAMETOOLONG name = name[0...-1] @@ -774,6 +780,9 @@ def main @decode_file_lock = Mutex.new profile_mem("#{group} start") puts "\nGetting articles for #{group}" +puts "nntpserver #{@config[group]["NNTPSERVER"]}" +puts "newsrcname #{@config[group]["NEWSRCNAME"]}" +puts "maxage #{@config[group]["MAXAGE"]}" @articles = Article.new(@config[group]["NNTPSERVER"], group, @config[group]["NEWSRCNAME"], @config[group]["MAXAGE"]) fill_preselector(group) puts "initialized"