ward/ripnews
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

11:28 <deBeuk> spam:

Browse source 
11:28 <deBeuk> ./news/article.rb:644:in `initialize': No such file or directory
               -
11:29 <deBeuk> +/mnt/news/News/.ripnews_caches/alt.binaries.multimedia.ripnewscache.new
11:29 <deBeuk> +szilla.xs4all.nl (Errno::ENOENT)
11:29 <deBeuk>         from ./news/article.rb:644:in `new'
11:29 <deBeuk>         from ./news/article.rb:644:in `cache_scrub'
11:29 <deBeuk>         from ./news/article.rb:593:in `cache_read'
11:29 <deBeuk>         from ./news/article.rb:592:in `each'
11:29 <deBeuk>         from ./news/article.rb:592:in `cache_read'
11:29 <deBeuk>         from ./news/article.rb:177:in `get_articles'
11:29 <deBeuk>         from ./ripnews.rb:605:in `main'
11:29 <deBeuk>         from ./ripnews.rb:597:in `each'
11:29 <deBeuk>         from ./ripnews.rb:597:in `main'
11:29 <deBeuk>         from ./ripnews.rb:668
12:22 <deBeuk> als je het mistte op #wizeazz: het gaat over een bug dat ripnews
               exit als een cachefile nog niet bestaat
This commit is contained in:
Ward Wouts 2004-11-21 14:21:03 +00:00
parent 745c164cec
commit c31095246e
1 changed files with 84 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

101
trunk/ripnews/news/article.rb
View file

@ -1,4 +1,4 @@
# $Dwarf: article.rb,v 1.92 2004/10/14 21:54:51 ward Exp $ # $Dwarf: article.rb,v 1.93 2004/10/15 13:48:04 ward Exp $
# $Source$ # $Source$
# #
@ -42,13 +42,13 @@ def initialize(nntpservers, groupname, newsrc="~/.newsrc")
@gotten = {} @gotten = {}
@group = groupname @group = groupname
@preselectpatterns = [] @preselectpatterns = []
@newids = {} @cache_buf = {}
@serverlist = nntpservers.split('|') @serverlist = nntpservers.split('|')
@connections = {} @connections = {}
@serverlist.collect{|server| @serverlist.collect{|server|
@connections[server] = {} @connections[server] = {}
@newids[server] = {} @cache_buf[server] = []
begin begin
p server p server
p Time.now p Time.now
@ -120,7 +120,7 @@ def memusage
end end
def add_preselect_pattern(regexp) def add_preselect_pattern(regexp)
@preselectpatterns.push(regexp) @preselectpatterns.push(Regexp.new(regexp))
end end
def preselect(subject) def preselect(subject)
@ -132,7 +132,7 @@ def preselect(subject)
return false return false
end end
def add(messid, id, server, subject) def add(id, messid, subject, server)
@messageinfo.push(Message.new(messid, id.to_i, server, subject)) @messageinfo.push(Message.new(messid, id.to_i, server, subject))
@grouped = false @grouped = false
end end
@ -145,7 +145,7 @@ end
def get_articles(cachedir=false) def get_articles(cachedir=false)
if cachedir != false if cachedir != false
check_cache(cachedir) cache_check(cachedir)
end end
for server in @connections.keys for server in @connections.keys
begin begin
@ -174,7 +174,7 @@ def get_articles(cachedir=false)
del_server(server) del_server(server)
end end
end end
read_cache(cachedir) cache_read(cachedir)
# for server in @connections.keys # for server in @connections.keys
# print "############################################################\n" # print "############################################################\n"
# print "skip_ids #{server}: #{@connections[server]["skip_ids"].run_list}\n" # print "skip_ids #{server}: #{@connections[server]["skip_ids"].run_list}\n"
@ -213,13 +213,20 @@ def get_articles(cachedir=false)
for id in art.keys for id in art.keys
if art[id].has_key?("subject") and art[id].has_key?("messid") if art[id].has_key?("subject") and art[id].has_key?("messid")
print "adding: #{art[id]["messid"]}, #{id}, #{server}, #{art[id]["subject"]}\n" if Debuglevel > 2 print "adding: #{art[id]["messid"]}, #{id}, #{server}, #{art[id]["subject"]}\n" if Debuglevel > 2
@newids[server][id.to_i] = true # @newids[server][id.to_i] = true
add(art[id]["messid"], id, server, art[id]["subject"]) # dit wellicht alleen doen indien preselector hem uitkiest
# en anders een leuk regeltje aan de cache toevoegen,
# maar niet in het geheugen houden
if preselect(art[id]["subject"])
add(id, art[id]["messid"], art[id]["subject"], server)
end
cache_add(cachedir, id, art[id]["messid"], art[id]["subject"], server)
end end
end end
headerlines += subj_lines.length headerlines += subj_lines.length
if headerlines >= 10000 # hmmm, dit lijkt niet te werken... #if headerlines >= 10000 # hmmm, dit lijkt niet te werken...
save_cache(cachedir, server) if headerlines >= 1000 # hmmm, dit lijkt niet te werken...
cache_save(cachedir, server)
headerlines = 0 headerlines = 0
end end
end end
@ -228,7 +235,7 @@ def get_articles(cachedir=false)
del_server(server) del_server(server)
next next
end end
save_cache(cachedir, server) cache_save(cachedir, server)
end end
GC.start GC.start
end end
@ -352,7 +359,7 @@ p "get_body"
end end
end end
return resp, id, messid, list return resp, id, messid, list
rescue TimeoutError rescue TimeoutError, Errno::ETIMEDOUT
print "Time out, reconnecting to server (get_body)\n" print "Time out, reconnecting to server (get_body)\n"
timedout += 1 timedout += 1
raise PermError, "Too many timeouts! (get_body)" if timedout > 1 raise PermError, "Too many timeouts! (get_body)" if timedout > 1
@ -558,13 +565,24 @@ def save_newsrc()
end end
end end
def check_cache(cachedir) def cache_add(cachedir, id, messid, subject, server)
if @cache_buf.has_key?(server)
@cache_buf[server].push("#{id}|#{messid}|#{subject}\n")
else
@cache_buf[server] = [ "#{id}|#{messid}|#{subject}\n" ]
end
if @cache_buf[server].length > 100
cache_save(cachedir, server)
end
end
def cache_check(cachedir)
if ! FileTest.exists?(cachedir) if ! FileTest.exists?(cachedir)
print "Cachedir '#{cachedir}' doesn't exists, performance will suffer\n" print "Cachedir '#{cachedir}' doesn't exists, performance will suffer\n"
end end
end end
def read_cache(cachedir) def cache_read(cachedir)
p "reading cache" p "reading cache"
p Time.now p Time.now
filename = "#{cachedir}/#{@group}.ripnewscache" filename = "#{cachedir}/#{@group}.ripnewscache"
@ -572,64 +590,49 @@ p Time.now
# id | messageid | subject # id | messageid | subject
lineregexp = Regexp.new('^(\d+)\|(.*?)\|(.*)$') lineregexp = Regexp.new('^(\d+)\|(.*?)\|(.*)$')
for server in @connections.keys for server in @connections.keys
cache_scrub(cachedir, server)
excludes[server] = {} excludes[server] = {}
@connections[server]["skip_ids"].elements.collect!{|x| excludes[server][x]=true} @connections[server]["skip_ids"].elements.collect!{|x| excludes[server][x]=true}
if FileTest.directory?( cachedir) and FileTest.file?( "#{filename}.#{server}" ) and FileTest.readable?( "#{filename}.#{server}" ) if FileTest.directory?( cachedir) and FileTest.file?( "#{filename}.#{server}" ) and FileTest.readable?( "#{filename}.#{server}" )
file = File.new( "#{filename}.#{server}" ) file = File.new( "#{filename}.#{server}" )
lines = file.readlines lines = file.readlines
#file.each{|line|
lines.collect{|line| lines.collect{|line|
# id | messageid | subject
#if line =~ /^(\d+)\|(.*?)\|(.*)$/
if line =~ lineregexp if line =~ lineregexp
#if lineregexp.match(line) != nil
unless excludes.has_key?(server) and excludes[server].has_key?($1.to_i) or unless excludes.has_key?(server) and excludes[server].has_key?($1.to_i) or
$1.to_i < @connections[server]["first"] or $1.to_i < @connections[server]["first"] or
$1.to_i > @connections[server]["last"] $1.to_i > @connections[server]["last"]
if preselect($3) if preselect($3)
add($2, $1, server, $3) add($1, $2, $3, server)
end end
@connections[server]["skip_ids"].insert($1.to_i) @connections[server]["skip_ids"].insert($1.to_i)
end end
end end
} }
file.close file.close
lines = []
end end
end end
p Time.now p Time.now
memusage memusage
end end
def save_cache(cachedir, server) def cache_save(cachedir, server)
p "writing cache" #p "writing cache"
p Time.now #p Time.now
filename = "#{cachedir}/#{@group}.ripnewscache" filename = "#{cachedir}/#{@group}.ripnewscache"
if FileTest.directory?( cachedir ) if FileTest.directory?( cachedir )
if ! File.copy("#{filename}.#{server}","#{filename}.#{server}.new") file = File.new( "#{filename}.#{server}", "a+" ) or print "couldn't open cachefile for writing\n"
puts "Couldn't renew cache" # print "Updating cache...\n"
end @cache_buf[server].sort!
file = File.new( "#{filename}.#{server}.new", "a+" ) or print "couldn't open cachefile for writing\n" file.print @cache_buf[server]
print "Updating cache...\n"
cache = []
for i in (0...@messageinfo.length)
if @newids[server].has_key?(@messageinfo[i][:id])
cache.push("#{@messageinfo[i][:id]}|#{@messageinfo[i][:messid]}|#{@messageinfo[i][:subject]}\n") if @messageinfo[i][:server] == server
end
end
cache.sort!
file.print cache
file.close file.close
if ( File.move("#{filename}.#{server}.new", "#{filename}.#{server}") ) @cache_buf[server] = []
print "Cache updated for #{server}\n" # print "Cache updated for #{server}\n"
else
print "Couldn't update #{server} cache\n"
end end
end #p Time.now
p Time.now
scrub_cache(cachedir, server)
end end
def scrub_cache(cachedir, server) def cache_scrub(cachedir, server)
# XXX this could and probably should be done in a separate thread... # XXX this could and probably should be done in a separate thread...
# XXX but it'll work for now # XXX but it'll work for now
# XXX also read articles aren't removed right now # XXX also read articles aren't removed right now
@ -637,6 +640,7 @@ def scrub_cache(cachedir, server)
p "scrubbing cache" p "scrubbing cache"
p Time.now p Time.now
filename = "#{cachedir}/#{@group}.ripnewscache" filename = "#{cachedir}/#{@group}.ripnewscache"
if File.exists?("#{filename}.#{server}")
regexp = Regexp.new('^(\d+)\|') regexp = Regexp.new('^(\d+)\|')
infile = File.new("#{filename}.#{server}") or puts "Couldn't open cachefile for reading" infile = File.new("#{filename}.#{server}") or puts "Couldn't open cachefile for reading"
outfile = File.new("#{filename}.#{server}.new", "w") or puts "Couldn't open cachefile for writing" outfile = File.new("#{filename}.#{server}.new", "w") or puts "Couldn't open cachefile for writing"
@ -653,6 +657,7 @@ p Time.now
else else
print "Couldn't scrub #{server} cache\n" print "Couldn't scrub #{server} cache\n"
end end
end
p Time.now p Time.now
end end
@ -692,6 +697,9 @@ p "pre sort length: #{@groups[subj]['messageinfo'].length}"
@groups[subj]["messageinfo"][i].dup @groups[subj]["messageinfo"][i].dup
) if serverhash[@groups[subj]["messageinfo"][i][:server]] != nil ) if serverhash[@groups[subj]["messageinfo"][i][:server]] != nil
end end
p "sort_arr length pre sort: #{sort_arr.length}"
if sort_arr.length != 0
sort_arr.sort!{|a,b| sort_arr.sort!{|a,b|
r = ward_sort(a[:subject], b[:subject]) r = ward_sort(a[:subject], b[:subject])
if serverhash[a[:server]] == nil or serverhash[b[:server]] == nil if serverhash[a[:server]] == nil or serverhash[b[:server]] == nil
@ -706,10 +714,11 @@ p "pre sort length: #{@groups[subj]['messageinfo'].length}"
end end
r r
} }
end
@groups[subj].clear @groups[subj].clear
@groups[subj]["total"] = total @groups[subj]["total"] = total
p "sort_arr length: #{sort_arr.length}" p "sort_arr length post sort: #{sort_arr.length}"
sort_arr.collect{|i| sort_arr.collect{|i|
if @groups[subj].has_key?("messageinfo") if @groups[subj].has_key?("messageinfo")
@groups[subj]["messageinfo"].push(i) @groups[subj]["messageinfo"].push(i)
@ -719,7 +728,9 @@ p "sort_arr length: #{sort_arr.length}"
print "subject sort: #{i[:subject]}\n" if Debuglevel > 2 print "subject sort: #{i[:subject]}\n" if Debuglevel > 2
print "server: #{i[:server]}\n" if Debuglevel > 2 print "server: #{i[:server]}\n" if Debuglevel > 2
} }
if ! @groups[subj]['messageinfo'].nil?
p "post sort length: #{@groups[subj]['messageinfo'].length}" p "post sort length: #{@groups[subj]['messageinfo'].length}"
end
#print "Done sorting\n" #print "Done sorting\n"
end end