Compare commits
No commits in common. "master" and "ripnews-20200312" have entirely different histories.
master
...
ripnews-20
3 changed files with 38 additions and 63 deletions
|
|
@ -90,8 +90,7 @@ def _uudecode_file(file, outfile)
|
|||
next if line =~ /[a-z]/
|
||||
next if line == nil
|
||||
next unless ((((line[0].ord - 32) & 077) + 2) / 3).to_i == (line.length/4).to_i
|
||||
#line.unpack("u").each{|x| outfile.print x}
|
||||
line.unpack("u").each{|x| outfile.write x}
|
||||
line.unpack("u").each{|x| outfile.print x}
|
||||
end
|
||||
|
||||
puts "No \"end\" found!!!"
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
# $Dwarf: article.rb,v 1.114 2005/05/12 07:39:53 ward Exp $
|
||||
# $Source$
|
||||
|
||||
#
|
||||
# Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 Ward Wouts <ward@wouts.nl>
|
||||
#
|
||||
|
|
@ -20,7 +23,7 @@ require Pathname.new(__FILE__).dirname + '../news/newsrc'
|
|||
require 'tempfile'
|
||||
require 'timeout'
|
||||
#require 'yaml'
|
||||
#require 'profiler' # removed from ruby?
|
||||
require 'profiler'
|
||||
|
||||
class ArticleError < RuntimeError; end
|
||||
class TempError < ArticleError; end
|
||||
|
|
@ -111,10 +114,6 @@ def initialize(nntpservers, groupname, newsrc="~/.newsrc", maxage=0)
|
|||
@serverlist = []
|
||||
@serverpasses = {}
|
||||
@maxage = maxage.to_i
|
||||
@oldestallowed = 0
|
||||
if @maxage != 0
|
||||
@oldestallowed = (DateTime.now - @maxage).strftime('%Y%m%d').to_i
|
||||
end
|
||||
|
||||
tmplist = nntpservers.split('|')
|
||||
tmplist.each{ |server|
|
||||
|
|
@ -149,7 +148,7 @@ def initialize(nntpservers, groupname, newsrc="~/.newsrc", maxage=0)
|
|||
# p server
|
||||
# p Time.now
|
||||
begin
|
||||
Timeout.timeout(60) do
|
||||
timeout(60) do
|
||||
#p "connecting"
|
||||
@connections[server]["nntp"] = Net::KANNTP.new(server, 119, @serverpasses[server]['user'], @serverpasses[server]['pass'])
|
||||
end
|
||||
|
|
@ -177,7 +176,7 @@ def reconnect(server)
|
|||
retries = 0
|
||||
begin
|
||||
puts "Trying to kill old connection #{Time.now}"
|
||||
Timeout.timeout(10) do
|
||||
timeout(10) do
|
||||
@connections[server]["nntp"].quit
|
||||
end
|
||||
puts "Killed old connection #{Time.now}"
|
||||
|
|
@ -189,7 +188,7 @@ def reconnect(server)
|
|||
puts "Trying to reconnect #{Time.now}"
|
||||
sleep 3
|
||||
#timeout(180) do
|
||||
Timeout.timeout(60) do
|
||||
timeout(60) do
|
||||
@connections[server]["nntp"] = Net::KANNTP.new(server, 119, @serverpasses[server]['user'], @serverpasses[server]['pass'])
|
||||
end
|
||||
resp = @connections[server]["nntp"].mode_reader
|
||||
|
|
@ -332,8 +331,7 @@ def get_articles(cachedir=false)
|
|||
art[id] = {} unless art.has_key?(id)
|
||||
|
||||
begin
|
||||
lastdate = DateTime.parse(date)
|
||||
art[id]["date"] = DateTime.parse(date).strftime('%Y%m%d')
|
||||
lastdate = art[id]["date"] = DateTime.parse(date).strftime('%Y%m%d')
|
||||
rescue
|
||||
puts $!.message
|
||||
puts id
|
||||
|
|
@ -356,16 +354,14 @@ def get_articles(cachedir=false)
|
|||
# dit wellicht alleen doen indien preselector hem uitkiest
|
||||
# en anders een leuk regeltje aan de cache toevoegen,
|
||||
# maar niet in het geheugen houden
|
||||
if art[id]["date"].to_i >= @oldestallowed && preselect(art[id]["subject"])
|
||||
if preselect(art[id]["subject"])
|
||||
add(id.to_i, art[id]["messid"], art[id]["date"], art[id]["from"], art[id]["subject"], server)
|
||||
end
|
||||
if art[id]["date"].to_i >= @oldestallowed
|
||||
cache_add(cachedir, id, art[id]["messid"], art[id]["date"], art[id]["from"], art[id]["subject"], server)
|
||||
end
|
||||
cache_add(cachedir, id, art[id]["messid"], art[id]["date"], art[id]["from"], art[id]["subject"], server)
|
||||
end
|
||||
}
|
||||
if @maxage and @maxage > 0
|
||||
if lastdate < ( DateTime.now - @maxage )
|
||||
if DateTime.parse(lastdate) < ( DateTime.now - @maxage )
|
||||
puts "Skipping articles older than #{DateTime.now - @maxage}"
|
||||
break
|
||||
end
|
||||
|
|
@ -388,7 +384,7 @@ def get_group_info(server)
|
|||
first = ""
|
||||
last = ""
|
||||
begin
|
||||
Timeout.timeout(30) do
|
||||
timeout(30) do
|
||||
begin
|
||||
resp, count, first, last, name = @connections[server]["nntp"].group(@group)
|
||||
rescue Net::NNTPReplyError
|
||||
|
|
@ -422,7 +418,7 @@ def get_xhdr(server, range, header)
|
|||
resp = ""
|
||||
lines = []
|
||||
begin
|
||||
Timeout.timeout(180) do
|
||||
timeout(180) do
|
||||
begin
|
||||
p Time.now if Debuglevel > 1
|
||||
puts "getting headers: #{header}, #{range}" if Debuglevel > 1
|
||||
|
|
@ -478,7 +474,7 @@ def get_xover(server, range)
|
|||
ed = start
|
||||
end
|
||||
begin
|
||||
Timeout.timeout(180) do
|
||||
timeout(180) do
|
||||
begin
|
||||
p Time.now if Debuglevel > 1
|
||||
puts "getting headers: #{range}" if Debuglevel > 1
|
||||
|
|
@ -536,7 +532,7 @@ def get_body(server, message)
|
|||
messid = ""
|
||||
list = []
|
||||
begin
|
||||
Timeout.timeout(180) do
|
||||
timeout(180) do
|
||||
begin
|
||||
list = []
|
||||
resp, id, messid, list = @connections[server]["nntp"].body(message)
|
||||
|
|
@ -829,7 +825,6 @@ def save_newsrc()
|
|||
end
|
||||
|
||||
def cache_add(cachedir, id, messid, date, from, subject, server)
|
||||
# also add to skip stuff
|
||||
if @cache_buf.has_key?(server)
|
||||
@cache_buf[server].push("#{id}|#{messid}|#{date}|#{from}|#{subject}\n")
|
||||
else
|
||||
|
|
@ -854,7 +849,7 @@ puts "#{Time.now} Reading & scrubbing caches"
|
|||
@connections.keys.each{|server|
|
||||
first = @connections[server]["first"]
|
||||
last = @connections[server]["last"]
|
||||
cache_scrub(cachedir, server)
|
||||
#cache_scrub(cachedir, server)
|
||||
puts " #{Time.now} Reading cache for #{server}"
|
||||
excludes[server] = {}
|
||||
@connections[server]["skip_ids"].elements.collect!{|x| excludes[server][x]=true}
|
||||
|
|
@ -881,13 +876,11 @@ puts " #{Time.now} Reading cache for #{server}"
|
|||
if first <= id_i and id_i <= last
|
||||
if ! excludes[server].has_key?(id_i)
|
||||
outfile.puts(line)
|
||||
if date.to_i >= @oldestallowed && preselect(subject)
|
||||
if preselect(subject)
|
||||
add(id_i, messid, date, from, subject, server)
|
||||
end
|
||||
# XXX alle traagheid van de cache_read zit in deze regel:
|
||||
if date.to_i < @oldestallowed
|
||||
@connections[server]["skip_ids"].insert!(id_i)
|
||||
end
|
||||
@connections[server]["skip_ids"].insert!(id_i)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -942,19 +935,11 @@ p Time.now
|
|||
outfile = File.new("#{filename}.#{server}.new", "w") or puts "Couldn't open cachefile for writing"
|
||||
infile.each{ |line|
|
||||
id, messid, date, subject = line.split("|", 3)
|
||||
#puts "#{date.to_i} #{@oldestallowed}"
|
||||
# XXX maybe also add to skipids ??
|
||||
next if date.to_i < @oldestallowed
|
||||
if id.to_i >= @connections[server]["first"] and
|
||||
id.to_i <= @connections[server]["last"]
|
||||
outfile.puts(line)
|
||||
end
|
||||
}
|
||||
if ( FileUtils.move("#{filename}.#{server}.new", "#{filename}.#{server}") )
|
||||
puts " #{Time.now} Cache scrubbed for #{server}"
|
||||
else
|
||||
puts "Couldn't scrub #{server} cache"
|
||||
end
|
||||
end
|
||||
p Time.now
|
||||
end
|
||||
|
|
|
|||
47
ripnews.rb
47
ripnews.rb
|
|
@ -50,20 +50,20 @@ def aprofile_mem(group)
|
|||
groups = {}
|
||||
ObjectSpace.each_object { |x|
|
||||
if not [Array,Hash].include? x.class
|
||||
e = nil
|
||||
begin
|
||||
e = MEntry.new( x.class, Marshal::dump(x).size )
|
||||
rescue TypeError # undumpable
|
||||
e = MEntry.new( x.class, 0 )
|
||||
end
|
||||
if groups.has_key? e.c
|
||||
groups[e.c].mem += e.mem
|
||||
groups[e.c].total += 1
|
||||
else
|
||||
groups[e.c] = GroupEntry.new( e.c, e.mem, 1 )
|
||||
end
|
||||
e = nil
|
||||
begin
|
||||
e = MEntry.new( x.class, Marshal::dump(x).size )
|
||||
rescue TypeError # undumpable
|
||||
e = MEntry.new( x.class, 0 )
|
||||
end
|
||||
}
|
||||
if groups.has_key? e.c
|
||||
groups[e.c].mem += e.mem
|
||||
groups[e.c].total += 1
|
||||
else
|
||||
groups[e.c] = GroupEntry.new( e.c, e.mem, 1 )
|
||||
end
|
||||
end
|
||||
}
|
||||
File.open( "mem_log", "a+" ) { |file|
|
||||
file << "Group #{group}\n"
|
||||
total = 0
|
||||
|
|
@ -547,17 +547,11 @@ def get_multi(subj, group)
|
|||
# puts "inside thread post pass\n"
|
||||
if UUEncode.is_uuencoded(tbody)
|
||||
puts " UUDecoding..."
|
||||
begin
|
||||
if tfile
|
||||
tmode, tfilename, tbody = UUEncode.uudecode(tfile, tfileout)
|
||||
else
|
||||
tmode, tfilename, tbody = UUEncode.uudecode(tbody)
|
||||
end
|
||||
rescue Encoding::UndefinedConversionError
|
||||
puts "#{$!}"
|
||||
puts " Skipping article..."
|
||||
Thread.current.exit
|
||||
end
|
||||
if tfile
|
||||
tmode, tfilename, tbody = UUEncode.uudecode(tfile, tfileout)
|
||||
else
|
||||
tmode, tfilename, tbody = UUEncode.uudecode(tbody)
|
||||
end
|
||||
elsif YEnc.is_yencoded(tbody)
|
||||
puts " YDecoding..."
|
||||
begin
|
||||
|
|
@ -689,7 +683,7 @@ def get_max_file_length(tempdir=".")
|
|||
name = "a"*500
|
||||
name = "#$$#{name}"
|
||||
begin
|
||||
File.new("#{tempdir}/#{name}", "w", 0644).close
|
||||
file = File.new("#{tempdir}/#{name}", "w", 0644).close
|
||||
FileUtils.rm("#{tempdir}/#{name}")
|
||||
rescue Errno::ENAMETOOLONG
|
||||
name = name[0...-1]
|
||||
|
|
@ -780,9 +774,6 @@ def main
|
|||
@decode_file_lock = Mutex.new
|
||||
profile_mem("#{group} start")
|
||||
puts "\nGetting articles for #{group}"
|
||||
puts "nntpserver #{@config[group]["NNTPSERVER"]}"
|
||||
puts "newsrcname #{@config[group]["NEWSRCNAME"]}"
|
||||
puts "maxage #{@config[group]["MAXAGE"]}"
|
||||
@articles = Article.new(@config[group]["NNTPSERVER"], group, @config[group]["NEWSRCNAME"], @config[group]["MAXAGE"])
|
||||
fill_preselector(group)
|
||||
puts "initialized"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue