ward/ripnews
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

be much more aggressive in scrubbing caches.

Browse source
This commit is contained in:
Ward Wouts 2021-04-01 14:35:36 +02:00
parent cde8e59f3e
commit 8c20f8435f
1 changed files with 22 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

23
news/article.rb
View file

@ -114,6 +114,10 @@ def initialize(nntpservers, groupname, newsrc="~/.newsrc", maxage=0)
@serverlist = [] @serverlist = []
@serverpasses = {} @serverpasses = {}
@maxage = maxage.to_i @maxage = maxage.to_i
@oldestallowed = 0
if @maxage != 0
@oldestallowed = (DateTime.now - @maxage).strftime('%Y%m%d').to_i
end
tmplist = nntpservers.split('|') tmplist = nntpservers.split('|')
tmplist.each{ |server| tmplist.each{ |server|
@ -355,11 +359,13 @@ def get_articles(cachedir=false)
# dit wellicht alleen doen indien preselector hem uitkiest # dit wellicht alleen doen indien preselector hem uitkiest
# en anders een leuk regeltje aan de cache toevoegen, # en anders een leuk regeltje aan de cache toevoegen,
# maar niet in het geheugen houden # maar niet in het geheugen houden
if preselect(art[id]["subject"]) if art[id]["date"].to_i >= @oldestallowed && preselect(art[id]["subject"])
add(id.to_i, art[id]["messid"], art[id]["date"], art[id]["from"], art[id]["subject"], server) add(id.to_i, art[id]["messid"], art[id]["date"], art[id]["from"], art[id]["subject"], server)
end end
if art[id]["date"].to_i >= @oldestallowed
cache_add(cachedir, id, art[id]["messid"], art[id]["date"], art[id]["from"], art[id]["subject"], server) cache_add(cachedir, id, art[id]["messid"], art[id]["date"], art[id]["from"], art[id]["subject"], server)
end end
end
} }
if @maxage and @maxage > 0 if @maxage and @maxage > 0
if lastdate < ( DateTime.now - @maxage ) if lastdate < ( DateTime.now - @maxage )
@ -826,6 +832,7 @@ def save_newsrc()
end end
def cache_add(cachedir, id, messid, date, from, subject, server) def cache_add(cachedir, id, messid, date, from, subject, server)
# also add to skip stuff
if @cache_buf.has_key?(server) if @cache_buf.has_key?(server)
@cache_buf[server].push("#{id}|#{messid}|#{date}|#{from}|#{subject}\n") @cache_buf[server].push("#{id}|#{messid}|#{date}|#{from}|#{subject}\n")
else else
@ -850,7 +857,7 @@ puts "#{Time.now} Reading & scrubbing caches"
@connections.keys.each{|server| @connections.keys.each{|server|
first = @connections[server]["first"] first = @connections[server]["first"]
last = @connections[server]["last"] last = @connections[server]["last"]
#cache_scrub(cachedir, server) cache_scrub(cachedir, server)
puts " #{Time.now} Reading cache for #{server}" puts " #{Time.now} Reading cache for #{server}"
excludes[server] = {} excludes[server] = {}
@connections[server]["skip_ids"].elements.collect!{|x| excludes[server][x]=true} @connections[server]["skip_ids"].elements.collect!{|x| excludes[server][x]=true}
@ -877,14 +884,16 @@ puts " #{Time.now} Reading cache for #{server}"
if first <= id_i and id_i <= last if first <= id_i and id_i <= last
if ! excludes[server].has_key?(id_i) if ! excludes[server].has_key?(id_i)
outfile.puts(line) outfile.puts(line)
if preselect(subject) if date.to_i >= @oldestallowed && preselect(subject)
add(id_i, messid, date, from, subject, server) add(id_i, messid, date, from, subject, server)
end end
# XXX alle traagheid van de cache_read zit in deze regel: # XXX alle traagheid van de cache_read zit in deze regel:
if date.to_i < @oldestallowed
@connections[server]["skip_ids"].insert!(id_i) @connections[server]["skip_ids"].insert!(id_i)
end end
end end
end end
end
rescue EOFError rescue EOFError
end end
if ( FileUtils.move("#{filename}.#{server}.new", "#{filename}.#{server}") ) if ( FileUtils.move("#{filename}.#{server}.new", "#{filename}.#{server}") )
@ -936,11 +945,19 @@ p Time.now
outfile = File.new("#{filename}.#{server}.new", "w") or puts "Couldn't open cachefile for writing" outfile = File.new("#{filename}.#{server}.new", "w") or puts "Couldn't open cachefile for writing"
infile.each{ |line| infile.each{ |line|
id, messid, date, subject = line.split("|", 3) id, messid, date, subject = line.split("|", 3)
#puts "#{date.to_i} #{@oldestallowed}"
# XXX maybe also add to skipids ??
next if date.to_i < @oldestallowed
if id.to_i >= @connections[server]["first"] and if id.to_i >= @connections[server]["first"] and
id.to_i <= @connections[server]["last"] id.to_i <= @connections[server]["last"]
outfile.puts(line) outfile.puts(line)
end end
} }
if ( FileUtils.move("#{filename}.#{server}.new", "#{filename}.#{server}") )
puts " #{Time.now} Cache scrubbed for #{server}"
else
puts "Couldn't scrub #{server} cache"
end
end end
p Time.now p Time.now
end end