diff --git a/CHANGELOG.md b/CHANGELOG.md index ec24c1e..a6cc72a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,3 @@ -ripnews-20200312 -- allow for different yenc line ordering - from 0.5.4 to ripnews-20200311 - add option to only rip one group - add option to list configured groups diff --git a/encode/uuencode.rb b/encode/uuencode.rb index 77c4f67..6058f65 100644 --- a/encode/uuencode.rb +++ b/encode/uuencode.rb @@ -90,8 +90,7 @@ def _uudecode_file(file, outfile) next if line =~ /[a-z]/ next if line == nil next unless ((((line[0].ord - 32) & 077) + 2) / 3).to_i == (line.length/4).to_i - #line.unpack("u").each{|x| outfile.print x} - line.unpack("u").each{|x| outfile.write x} + line.unpack("u").each{|x| outfile.print x} end puts "No \"end\" found!!!" diff --git a/encode/yenc.rb b/encode/yenc.rb index 8424a9e..b039b57 100644 --- a/encode/yenc.rb +++ b/encode/yenc.rb @@ -97,14 +97,12 @@ def _ydecode_file(file, outfile) if line.match(/^\=ybegin\s+(.*line\=.*)/) m = $1 puts " #{Thread.current.inspect} ybegin match; rest: #{m}" if Debuglevel > 0 - print "full line: #{line}" if Debuglevel > 0 - if matchdata = m.match(/^\s*(part\=(?\d+)\s+)?(total\=(?\d+)\s+)?(line\=(?\d+))(\s*size\=(?\d+))(\s*name=(?.*?\S))\s*$/) or - matchdata = m.match(/^\s*(part\=(?\d+)\s+)?(total\=(?\d+)\s+)?(size\=(?\d+))(\s*line\=(?\d+))(\s*name=(?.*?\S))\s*$/) - part = matchdata[:part].to_i - total = matchdata[:total].to_i - linesize = matchdata[:linesize].to_i - totalsize = matchdata[:totalsize].to_i - filename = matchdata[:filename] + if m.match(/^\s*(part\=(\d+)\s+)?(total\=(\d+)\s+)?(line\=(\d+))(\s*size\=(\d+))(\s*name=(.*?\S))\s*$/) + part = $2.to_i + total = $4.to_i + linesize = $6.to_i + totalsize = $8.to_i + filename = $10 if Debuglevel > 0 print "found beginning" if part != nil @@ -163,13 +161,12 @@ def _ydecode_file(file, outfile) closure = false m = $1 search_begin = false - if matchdata = m.match(/^\s*(part\=(?\d+)\s+)?(total\=(?\d+)\s+)?(line\=(?\d+))(\s*size\=(?\d+))(\s*name=(?.*?\S))\s*$/) or - matchdata = m.match(/^\s*(part\=(?\d+)\s+)?(total\=(?\d+)\s+)?(size\=(?\d+))(\s*line\=(?\d+))(\s*name=(?.*?\S))\s*$/) - part = matchdata[:part].to_i - total = matchdata[:total].to_i - linesize = matchdata[:linesize].to_i - totalsize = matchdata[:totalsize].to_i - filename = matchdata[:filename] + if m.match(/^\s*(part\=(\d+)\s+)?(total\=(\d+)\s+)?(line\=(\d+))(\s*size\=(\d+))(\s*name=(.*?\S))\s*$/) + part = $2.to_i + total = $4.to_i + linesize = $6.to_i + totalsize = $8.to_i + filename = $10 puts "found beginning of part #{part}, linesize = #{linesize}, size = #{totalsize}, filename = #{filename}" if Debuglevel > 0 end next @@ -233,14 +230,13 @@ def _ydecode_array(data) if data[i].match(/^\=ybegin\s+(.*line\=.*)/) m = $1 puts " #{Thread.current.inspect} ybegin match; rest: #{m}" if Debuglevel > 0 - if matchdata = m.match(/^\s*(part\=(?\d+)\s+)?(total\=(?\d+)\s+)?(line\=(?\d+))(\s*size\=(?\d+))(\s*name=(?.*?\S))\s*$/) or - matchdata = m.match(/^\s*(part\=(?\d+)\s+)?(total\=(?\d+)\s+)?(size\=(?\d+))(\s*line\=(?\d+))(\s*name=(?.*?\S))\s*$/) - part = matchdata[:part].to_i - total = matchdata[:total].to_i - linesize = matchdata[:linesize].to_i - totalsize = matchdata[:totalsize].to_i - filename = matchdata[:filename] - puts " #{Thread.current.inspect} found beginning, linesize = #{linesize}, size = #{totalsize}, filename = #{filename}" if Debuglevel > 0 + if m.match(/^\s*(part\=(\d+)\s+)?(total\=(\d+)\s+)?(line\=(\d+))(\s*size\=(\d+))(\s*name=(.*?\S))\s*$/) + part = $2.to_i + total = $4.to_i + linesize = $6.to_i + size = $8.to_i + filename = $10 + puts " #{Thread.current.inspect} found beginning, linesize = #{linesize}, size = #{size}, filename = #{filename}" if Debuglevel > 0 i += 1 break else @@ -321,9 +317,8 @@ def get_filename(data) i = 0 while i < data.length line = data[i] - if matchdata = line.match(/=ybegin\s*(part\=(\d+)\s+)?(total\=(\d+)\s+)?(line\=(\d+))(\s*size\=(\d+))(\s*name=(?.*?\S))\s*$/m) or - matchdata = line.match(/=ybegin\s*(part\=(\d+)\s+)?(total\=(\d+)\s+)?(size\=(\d+))(\s*line\=(\d+))(\s*name=(?.*?\S))\s*$/m) - return matchdata[:filename] + if line.match(/=ybegin\s*(part\=(\d+)\s+)?(total\=(\d+)\s+)?(line\=(\d+))(\s*size\=(\d+))(\s*name=(.*?\S))\s*$/m) + return $10 end i += 1 end diff --git a/news/article.rb b/news/article.rb index 8118343..cbfe52a 100644 --- a/news/article.rb +++ b/news/article.rb @@ -1,3 +1,6 @@ +# $Dwarf: article.rb,v 1.114 2005/05/12 07:39:53 ward Exp $ +# $Source$ + # # Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 Ward Wouts # @@ -20,7 +23,7 @@ require Pathname.new(__FILE__).dirname + '../news/newsrc' require 'tempfile' require 'timeout' #require 'yaml' -#require 'profiler' # removed from ruby? +require 'profiler' class ArticleError < RuntimeError; end class TempError < ArticleError; end @@ -111,10 +114,6 @@ def initialize(nntpservers, groupname, newsrc="~/.newsrc", maxage=0) @serverlist = [] @serverpasses = {} @maxage = maxage.to_i - @oldestallowed = 0 - if @maxage != 0 - @oldestallowed = (DateTime.now - @maxage).strftime('%Y%m%d').to_i - end tmplist = nntpservers.split('|') tmplist.each{ |server| @@ -149,7 +148,7 @@ def initialize(nntpservers, groupname, newsrc="~/.newsrc", maxage=0) # p server # p Time.now begin - Timeout.timeout(60) do + timeout(60) do #p "connecting" @connections[server]["nntp"] = Net::KANNTP.new(server, 119, @serverpasses[server]['user'], @serverpasses[server]['pass']) end @@ -177,7 +176,7 @@ def reconnect(server) retries = 0 begin puts "Trying to kill old connection #{Time.now}" - Timeout.timeout(10) do + timeout(10) do @connections[server]["nntp"].quit end puts "Killed old connection #{Time.now}" @@ -189,7 +188,7 @@ def reconnect(server) puts "Trying to reconnect #{Time.now}" sleep 3 #timeout(180) do - Timeout.timeout(60) do + timeout(60) do @connections[server]["nntp"] = Net::KANNTP.new(server, 119, @serverpasses[server]['user'], @serverpasses[server]['pass']) end resp = @connections[server]["nntp"].mode_reader @@ -332,8 +331,7 @@ def get_articles(cachedir=false) art[id] = {} unless art.has_key?(id) begin - lastdate = DateTime.parse(date) - art[id]["date"] = DateTime.parse(date).strftime('%Y%m%d') + lastdate = art[id]["date"] = DateTime.parse(date).strftime('%Y%m%d') rescue puts $!.message puts id @@ -356,16 +354,14 @@ def get_articles(cachedir=false) # dit wellicht alleen doen indien preselector hem uitkiest # en anders een leuk regeltje aan de cache toevoegen, # maar niet in het geheugen houden - if art[id]["date"].to_i >= @oldestallowed && preselect(art[id]["subject"]) + if preselect(art[id]["subject"]) add(id.to_i, art[id]["messid"], art[id]["date"], art[id]["from"], art[id]["subject"], server) end - if art[id]["date"].to_i >= @oldestallowed - cache_add(cachedir, id, art[id]["messid"], art[id]["date"], art[id]["from"], art[id]["subject"], server) - end + cache_add(cachedir, id, art[id]["messid"], art[id]["date"], art[id]["from"], art[id]["subject"], server) end } if @maxage and @maxage > 0 - if lastdate < ( DateTime.now - @maxage ) + if DateTime.parse(lastdate) < ( DateTime.now - @maxage ) puts "Skipping articles older than #{DateTime.now - @maxage}" break end @@ -388,7 +384,7 @@ def get_group_info(server) first = "" last = "" begin - Timeout.timeout(30) do + timeout(30) do begin resp, count, first, last, name = @connections[server]["nntp"].group(@group) rescue Net::NNTPReplyError @@ -422,7 +418,7 @@ def get_xhdr(server, range, header) resp = "" lines = [] begin - Timeout.timeout(180) do + timeout(180) do begin p Time.now if Debuglevel > 1 puts "getting headers: #{header}, #{range}" if Debuglevel > 1 @@ -478,7 +474,7 @@ def get_xover(server, range) ed = start end begin - Timeout.timeout(180) do + timeout(180) do begin p Time.now if Debuglevel > 1 puts "getting headers: #{range}" if Debuglevel > 1 @@ -536,7 +532,7 @@ def get_body(server, message) messid = "" list = [] begin - Timeout.timeout(180) do + timeout(180) do begin list = [] resp, id, messid, list = @connections[server]["nntp"].body(message) @@ -829,7 +825,6 @@ def save_newsrc() end def cache_add(cachedir, id, messid, date, from, subject, server) - # also add to skip stuff if @cache_buf.has_key?(server) @cache_buf[server].push("#{id}|#{messid}|#{date}|#{from}|#{subject}\n") else @@ -854,7 +849,7 @@ puts "#{Time.now} Reading & scrubbing caches" @connections.keys.each{|server| first = @connections[server]["first"] last = @connections[server]["last"] - cache_scrub(cachedir, server) + #cache_scrub(cachedir, server) puts " #{Time.now} Reading cache for #{server}" excludes[server] = {} @connections[server]["skip_ids"].elements.collect!{|x| excludes[server][x]=true} @@ -881,13 +876,11 @@ puts " #{Time.now} Reading cache for #{server}" if first <= id_i and id_i <= last if ! excludes[server].has_key?(id_i) outfile.puts(line) - if date.to_i >= @oldestallowed && preselect(subject) + if preselect(subject) add(id_i, messid, date, from, subject, server) end # XXX alle traagheid van de cache_read zit in deze regel: - if date.to_i < @oldestallowed - @connections[server]["skip_ids"].insert!(id_i) - end + @connections[server]["skip_ids"].insert!(id_i) end end end @@ -942,19 +935,11 @@ p Time.now outfile = File.new("#{filename}.#{server}.new", "w") or puts "Couldn't open cachefile for writing" infile.each{ |line| id, messid, date, subject = line.split("|", 3) - #puts "#{date.to_i} #{@oldestallowed}" - # XXX maybe also add to skipids ?? - next if date.to_i < @oldestallowed if id.to_i >= @connections[server]["first"] and id.to_i <= @connections[server]["last"] outfile.puts(line) end } - if ( FileUtils.move("#{filename}.#{server}.new", "#{filename}.#{server}") ) - puts " #{Time.now} Cache scrubbed for #{server}" - else - puts "Couldn't scrub #{server} cache" - end end p Time.now end diff --git a/ripnews.rb b/ripnews.rb index 2af18c7..1f81e12 100755 --- a/ripnews.rb +++ b/ripnews.rb @@ -50,20 +50,20 @@ def aprofile_mem(group) groups = {} ObjectSpace.each_object { |x| if not [Array,Hash].include? x.class - e = nil - begin - e = MEntry.new( x.class, Marshal::dump(x).size ) - rescue TypeError # undumpable - e = MEntry.new( x.class, 0 ) - end - if groups.has_key? e.c - groups[e.c].mem += e.mem - groups[e.c].total += 1 - else - groups[e.c] = GroupEntry.new( e.c, e.mem, 1 ) - end + e = nil + begin + e = MEntry.new( x.class, Marshal::dump(x).size ) + rescue TypeError # undumpable + e = MEntry.new( x.class, 0 ) end - } + if groups.has_key? e.c + groups[e.c].mem += e.mem + groups[e.c].total += 1 + else + groups[e.c] = GroupEntry.new( e.c, e.mem, 1 ) + end + end + } File.open( "mem_log", "a+" ) { |file| file << "Group #{group}\n" total = 0 @@ -547,17 +547,11 @@ def get_multi(subj, group) # puts "inside thread post pass\n" if UUEncode.is_uuencoded(tbody) puts " UUDecoding..." - begin - if tfile - tmode, tfilename, tbody = UUEncode.uudecode(tfile, tfileout) - else - tmode, tfilename, tbody = UUEncode.uudecode(tbody) - end - rescue Encoding::UndefinedConversionError - puts "#{$!}" - puts " Skipping article..." - Thread.current.exit - end + if tfile + tmode, tfilename, tbody = UUEncode.uudecode(tfile, tfileout) + else + tmode, tfilename, tbody = UUEncode.uudecode(tbody) + end elsif YEnc.is_yencoded(tbody) puts " YDecoding..." begin @@ -689,7 +683,7 @@ def get_max_file_length(tempdir=".") name = "a"*500 name = "#$$#{name}" begin - File.new("#{tempdir}/#{name}", "w", 0644).close + file = File.new("#{tempdir}/#{name}", "w", 0644).close FileUtils.rm("#{tempdir}/#{name}") rescue Errno::ENAMETOOLONG name = name[0...-1] @@ -780,9 +774,6 @@ def main @decode_file_lock = Mutex.new profile_mem("#{group} start") puts "\nGetting articles for #{group}" -puts "nntpserver #{@config[group]["NNTPSERVER"]}" -puts "newsrcname #{@config[group]["NEWSRCNAME"]}" -puts "maxage #{@config[group]["MAXAGE"]}" @articles = Article.new(@config[group]["NNTPSERVER"], group, @config[group]["NEWSRCNAME"], @config[group]["MAXAGE"]) fill_preselector(group) puts "initialized"