ward/ripnews
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

change some debuglevels, fix rechunk_runlist, use temporary file writing cache

Browse source
This commit is contained in:
Ward Wouts 2003-07-13 09:52:53 +00:00
parent 9b82606a4e
commit 8b0eb20bdc
1 changed files with 50 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

68
trunk/ripnews/news/article.rb
View file

@ -1,6 +1,6 @@
################################# #################################
# #
# $Dwarf: article.rb,v 1.69 2003/07/03 14:03:11 ward Exp $ # $Dwarf: article.rb,v 1.70 2003/07/06 08:14:05 ward Exp $
# $Source$ # $Source$
# #
# article.rb # article.rb
@ -21,7 +21,7 @@ class PermError < ArticleError; end
class Article class Article
Debuglevel = 0 Debuglevel = 2
def initialize(nntpservers, groupname, newsrc="~/.newsrc") def initialize(nntpservers, groupname, newsrc="~/.newsrc")
@messids = [] @messids = []
@ -44,7 +44,7 @@ def initialize(nntpservers, groupname, newsrc="~/.newsrc")
@connections[server]["skip_ids"] = Set::IntSpan.new() @connections[server]["skip_ids"] = Set::IntSpan.new()
@connections[server]["newsrc"] = News::Newsrc.new("#{newsrc}.#{server}") @connections[server]["newsrc"] = News::Newsrc.new("#{newsrc}.#{server}")
set_skip_ids(server, @connections[server]["newsrc"].marked_articles(@group)) set_skip_ids(server, @connections[server]["newsrc"].marked_articles(@group))
rescue SocketError, Errno::EINVAL rescue SocketError, Errno::EINVAL, EOFError
print "Connection to #{server} failed: #{$!}\n" print "Connection to #{server} failed: #{$!}\n"
del_server(server) del_server(server)
end end
@ -54,7 +54,7 @@ end
def reconnect(server) def reconnect(server)
begin begin
@connections[server]["nntp"] = Net::NNTP.new(server) @connections[server]["nntp"] = Net::NNTP.new(server)
rescue SocketError, Errno::EINVAL rescue SocketError, Errno::EINVAL, EOFError
print "Reconnect to #{server} failed: #{$!}\n" print "Reconnect to #{server} failed: #{$!}\n"
del_server(server) del_server(server)
raise PermError, "Couldn't connect to #{server}" raise PermError, "Couldn't connect to #{server}"
@ -93,9 +93,10 @@ def get_articles(cachedir=false)
del_server(server) del_server(server)
next next
end end
if first <= last if first.to_i <= last.to_i
@connections[server]["first"] = first ? first : 0 # available articles on server
@connections[server]["last"] = last ? last : 0 @connections[server]["first"] = first ? first.to_i : 0
@connections[server]["last"] = last ? last.to_i : 0
if Debuglevel > 0 if Debuglevel > 0
print " Server: #{server}\n" print " Server: #{server}\n"
print " First: #{first}\n" print " First: #{first}\n"
@ -112,13 +113,17 @@ def get_articles(cachedir=false)
end end
end end
read_cache(cachedir) read_cache(cachedir)
for server in @connections.keys
print "############################################################\n"
print "skip_ids #{server}: #{@connections[server]["skip_ids"].run_list}\n"
end
for server in @connections.keys for server in @connections.keys
print " reading articles from server: #{server}\n" print " reading articles from server: #{server}\n"
range = Set::IntSpan.new("#{@connections[server]["first"]}-#{@connections[server]["last"]}") range = Set::IntSpan.new("#{@connections[server]["first"]}-#{@connections[server]["last"]}")
rangelist = rechunk_runlist(range.diff(@connections[server]["skip_ids"]).run_list) rangelist = rechunk_runlist(range.diff(@connections[server]["skip_ids"]).run_list)
print "rangelist: #{rangelist}\n" if Debuglevel > 2 print "rangelist: #{rangelist}\n" if Debuglevel > 2
print "rangelist: #{rangelist.type.to_s}\n" if Debuglevel > 2 print "rangelist: #{rangelist.type.to_s}\n" if Debuglevel > 2
print "rangelist elements: #{range.diff(@connections[server]["skip_ids"]).elements}\n" if Debuglevel >2 print "rangelist elements: #{range.diff(@connections[server]["skip_ids"]).elements}\n" if Debuglevel > 2
begin begin
unless rangelist == nil or rangelist =~ /^$/ unless rangelist == nil or rangelist =~ /^$/
for i in rangelist.split(',') for i in rangelist.split(',')
@ -135,16 +140,16 @@ def get_articles(cachedir=false)
subj_lines.collect{|x| subj_lines.collect{|x|
art[x[0]] = {} unless art.has_key?(x[0]) art[x[0]] = {} unless art.has_key?(x[0])
art[x[0]]["subject"] = x[1] art[x[0]]["subject"] = x[1]
print "art id: #{x[0]} subj: #{x[1]}\n" if Debuglevel > 1 print "art id: #{x[0]} subj: #{x[1]}\n" if Debuglevel > 2
} }
messid_lines.collect{|x| messid_lines.collect{|x|
art[x[0]] = {} unless art.has_key?(x[0]) art[x[0]] = {} unless art.has_key?(x[0])
art[x[0]]["messid"] = x[1] art[x[0]]["messid"] = x[1]
print "art id: #{x[0]} messid: #{x[1]}\n" if Debuglevel > 1 print "art id: #{x[0]} messid: #{x[1]}\n" if Debuglevel > 2
} }
for id in art.keys for id in art.keys
if art[id].has_key?("subject") and art[id].has_key?("messid") if art[id].has_key?("subject") and art[id].has_key?("messid")
print "adding: #{art[id]["messid"]}, #{id}, #{server}, #{art[id]["subject"]}\n" if Debuglevel > 1 print "adding: #{art[id]["messid"]}, #{id}, #{server}, #{art[id]["subject"]}\n" if Debuglevel > 2
add(art[id]["messid"], id, server, art[id]["subject"]) add(art[id]["messid"], id, server, art[id]["subject"])
end end
end end
@ -195,6 +200,7 @@ def get_xhdr(server, range, header)
begin begin
timeout(180) do timeout(180) do
begin begin
print "getting headers: #{header}, #{range}\n" if Debuglevel > 1
resp, lines = @connections[server]["nntp"].xhdr(header, range) resp, lines = @connections[server]["nntp"].xhdr(header, range)
if resp.to_i == 500 if resp.to_i == 500
print "xhdr not implemented\n" print "xhdr not implemented\n"
@ -212,7 +218,7 @@ def get_xhdr(server, range, header)
get_group_info(server) get_group_info(server)
retry retry
else else
print "Won't handdle this... yet :(\n" print "Won't handle this... yet :(\n"
end end
rescue Errno::EPIPE, Errno::ECONNRESET, EOFError rescue Errno::EPIPE, Errno::ECONNRESET, EOFError
print "Caught #{$!.type} reading from server #{server} (get_xhdr)\n" print "Caught #{$!.type} reading from server #{server} (get_xhdr)\n"
@ -285,7 +291,7 @@ def get_body(server, message)
retry retry
end end
return false return false
rescue EOFError rescue EOFError, NameError
print "Caught #{$!.type} reading article #{message} from #{server} (get_body)\n" print "Caught #{$!.type} reading article #{message} from #{server} (get_body)\n"
print "Error: #{$!}\n" print "Error: #{$!}\n"
return false return false
@ -453,7 +459,7 @@ end
def group_subjects def group_subjects
@groups = {} @groups = {}
for i in (0...@subjects.length) for i in (0...@subjects.length)
print "group subjects: #{i} #{@subjects[i]}\n" if Debuglevel > 1 print "group subjects: #{i} #{@subjects[i]}\n" if Debuglevel > 3
if @subjects[i] =~ /(.*)\((\d+)\/(\d+)\)(.*)/ || @subjects[i] =~ /(.*)\[(\d+)\/(\d+)\](.*)/ if @subjects[i] =~ /(.*)\((\d+)\/(\d+)\)(.*)/ || @subjects[i] =~ /(.*)\[(\d+)\/(\d+)\](.*)/
j = "#{$1}#{$4} (#{$3})" j = "#{$1}#{$4} (#{$3})"
number = $2 number = $2
@ -490,9 +496,12 @@ def set_skip_ids(server, ids)
end end
def group_update_newsrc(subject) def group_update_newsrc(subject)
print "running group_update_newsrc\n";
for i in (0...@groups[subject]["messages"].length) for i in (0...@groups[subject]["messages"].length)
if @connections[@groups[subject]["servers"][i]]
@connections[@groups[subject]["servers"][i]]["newsrc"].mark(@group, @groups[subject]["ids"][i]) @connections[@groups[subject]["servers"][i]]["newsrc"].mark(@group, @groups[subject]["ids"][i])
end end
end
end end
def save_newsrc() def save_newsrc()
@ -536,7 +545,7 @@ end
def save_cache(cachedir) def save_cache(cachedir)
filename = "#{cachedir}/#{@group}.ripnewscache" filename = "#{cachedir}/#{@group}.ripnewscache"
if FileTest.directory?( cachedir ) if FileTest.directory?( cachedir )
file = File.new( filename, "w" ) or print "couldn't open cachefile for writing\n" file = File.new( "#{filename}.new", "w" ) or print "couldn't open cachefile for writing\n"
cache = [] cache = []
for i in (0...@subjects.length) for i in (0...@subjects.length)
cache.push("#{@ids[i]}|#{@messids[i]}|#{@servers[i]}|#{@subjects[i]}\n") cache.push("#{@ids[i]}|#{@messids[i]}|#{@servers[i]}|#{@subjects[i]}\n")
@ -544,6 +553,11 @@ def save_cache(cachedir)
cache.sort! cache.sort!
file.print cache file.print cache
file.close file.close
if ( File.move("#{filename}.new", filename) )
print "Cache updated\n"
else
print "Couldn't update cache\n"
end
end end
end end
@ -585,6 +599,13 @@ def group_subject_sort(subj)
end end
sort_arr.sort!{|a,b| sort_arr.sort!{|a,b|
r = ward_sort(a[0], b[0]) r = ward_sort(a[0], b[0])
if serverhash[a[3]] == nil or serverhash[b[3]] == nil
print "serverhash[a[3]]: #{serverhash[a[3]]}\n"
print "serverhash[b[3]]: #{serverhash[b[3]]}\n"
print "a[3]: #{a[3]}\n"
print "b[3]: #{a[3]}\n"
print "strange things going on here...\n"
end
if r == 0 if r == 0
r = serverhash[a[3]] <=> serverhash[b[3]] r = serverhash[a[3]] <=> serverhash[b[3]]
end end
@ -628,23 +649,30 @@ end
def rechunk_runlist(runlist) def rechunk_runlist(runlist)
return nil if runlist == nil return nil if runlist == nil
chunksize = 500
blalist = runlist.split(',') blalist = runlist.split(',')
# hmmm, als het aantal articles wat tussen de komma's ligt < pak um beet 3
# dan is het volgens mij heel erg de moeite die 3 ook gewoon binnen te halen
# en minder network requests te doen...
# de manier om dat te doen is dan iets van die komma weghalen en
# een van de 2 getallen...
blalist.collect!{|x| blalist.collect!{|x|
result = "" result = ""
if x =~ /(.*)-(.*)/ if x =~ /(.*)-(.*)/
a = $1 a = $1
while ($2.to_i - a.to_i) > 200 while ($2.to_i - a.to_i) > chunksize
result << "#{a}-#{a.to_i+199}," result << "#{a}-#{a.to_i+(chunksize-1)},"
a = a.to_i + 200 a = a.to_i + chunksize
end end
result << "#{a}-#{$2}" result << "#{a}-#{$2}"
else else
x x
end end
}
blup = blalist.join(",") blup = blalist.join(",")
return blup return blup
}
return
end end
def quit def quit