ward/ripnews
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

optimizations and new aging option

Browse source
This commit is contained in:
Ward Wouts 2008-07-28 19:51:15 +00:00
parent f18662baef
commit f181edab96
4 changed files with 115 additions and 76 deletions
Download patch file Download diff file Expand all files Collapse all files

1
trunk/ripnews/README
View file

@ -135,6 +135,7 @@ OPT_MR=<pattern> Set "mark read" pattern. Ripnews will place
again. Great for reducing memory usage when again. Great for reducing memory usage when
checking a group for the first time. checking a group for the first time.
OPT_MRF=<patter> Set "mark read from" pattern. Filters on posters. OPT_MRF=<patter> Set "mark read from" pattern. Filters on posters.
OPT_MRO=<days> Set "mark read old". Filters posts older that set days.
OPT_MRR=<bool> Mark Remaining Read. If this is set to OPT_MRR=<bool> Mark Remaining Read. If this is set to
true and the article doesn't match an exclude or true and the article doesn't match an exclude or
include pattern, the article will be include pattern, the article will be

4
trunk/ripnews/TODO
View file

@ -26,7 +26,5 @@
added to the main buffer added to the main buffer
[ ] running without a tempdir doesn't work at all [ ] running without a tempdir doesn't work at all
[ ] don't drop connections to servers when switching groups [ ] don't drop connections to servers when switching groups
[ ] keep connections to newsservers alive (don't timeout)
[ ] a markread mechanism for every post that is not specifically
included, this could be used to prevent excessively big caches
[ ] limiter on running time. if running longer than X, abort [ ] limiter on running time. if running longer than X, abort
[ ] implement article expiration

17
trunk/ripnews/news/article.rb
View file

@ -2,7 +2,7 @@
# $Source$ # $Source$
# #
# Copyright (c) 2002, 2003, 2004, 2005 Ward Wouts <ward@wouts.nl> # Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 Ward Wouts <ward@wouts.nl>
# #
# Permission to use, copy, modify, and distribute this software for any # Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above # purpose with or without fee is hereby granted, provided that the above
@ -236,7 +236,7 @@ def preselect(subject)
end end
def add(id, messid, date, from, subject, server) def add(id, messid, date, from, subject, server)
@messageinfo.push(Message.new(messid, id, date, from, server, subject)) @messageinfo.push(Message.new(messid, id.to_i, date.to_i, from, server, subject))
@grouped = false @grouped = false
end end
@ -681,6 +681,15 @@ def get_group_poster(subj)
return @groups[subj]["messageinfo"][0][:from] return @groups[subj]["messageinfo"][0][:from]
end end
def get_group_date(subj)
group_subject_sort(subj)
unless @groups[subj]["messageinfo"] != nil && @groups[subj]["messageinfo"][0][:date]
p "ieks komt niet door lame check heen"
return false
end
return @groups[subj]["messageinfo"][0][:date]
end
def group_is_complete(subj) def group_is_complete(subj)
group_subjects unless @grouped group_subjects unless @grouped
#print "Subject: #{subj}\n" #print "Subject: #{subj}\n"
@ -823,7 +832,7 @@ puts "#{Time.now} Reading & scrubbing caches"
puts " #{Time.now} Reading cache for #{server}" puts " #{Time.now} Reading cache for #{server}"
excludes[server] = {} excludes[server] = {}
@connections[server]["skip_ids"].elements.collect!{|x| excludes[server][x]=true} @connections[server]["skip_ids"].elements.collect!{|x| excludes[server][x]=true}
if FileTest.directory?( cachedir) and FileTest.file?( "#{filename}.#{server}" ) and FileTest.readable?( "#{filename}.#{server}" ) if FileTest.directory?(cachedir) and FileTest.file?( "#{filename}.#{server}" ) and FileTest.readable?( "#{filename}.#{server}" )
outfile = File.new("#{filename}.#{server}.new", "w") or puts "Couldn't open cachefile for writing" outfile = File.new("#{filename}.#{server}.new", "w") or puts "Couldn't open cachefile for writing"
File.new( "#{filename}.#{server}" ).each{ |line| File.new( "#{filename}.#{server}" ).each{ |line|
line =~ regexp line =~ regexp
@ -833,7 +842,7 @@ puts " #{Time.now} Reading cache for #{server}"
from = $4 from = $4
subject = $5 subject = $5
if first <= id_i and id_i <= last if first <= id_i and id_i <= last
unless excludes[server].has_key?(id_i) if ! excludes[server].has_key?(id_i)
outfile.puts(line) outfile.puts(line)
if preselect(subject) if preselect(subject)
add(id_i, messid, date, from, subject, server) add(id_i, messid, date, from, subject, server)

169
trunk/ripnews/ripnews.rb
View file

@ -4,7 +4,7 @@
# $Source$ # $Source$
# #
# Copyright (c) 2002, 2003, 2004, 2005 Ward Wouts <ward@wouts.nl> # Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 Ward Wouts <ward@wouts.nl>
# #
# Permission to use, copy, modify, and distribute this software for any # Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above # purpose with or without fee is hereby granted, provided that the above
@ -83,72 +83,8 @@ Debuglevel = 0
def save_file(dir, name, data) def save_file(dir, name, data)
puts "savename: #{name}" if Debuglevel > 1 puts "savename: #{name}" if Debuglevel > 1
nname = name.gsub(/\//, "-") nname = filename_sanitize(name)
nname.gsub!(/>/, "")
nname.gsub!(/</, "")
nname.gsub!(/©/, "(c)")
nname.gsub!(/®/, "(r)")
# XXX I really have to find something a bit more elegant for this
# XXX this is a mac problem, should be configurable!!!!
group = @config.keys[0]
if @config[group].has_key?("ASCIIFILENAMES") && @config[group]["ASCIIFILENAMES"]
require 'iconv'
begin
nname = Iconv.conv("ASCII//TRANSLIT", "UTF-8", nname)
rescue Iconv::IllegalSequence
puts "Iconv couldn't handle: '#{nname}'"
nname.gsub!(/\\\d+/, "#")
nname.gsub!(/ä/, "\"a")
nname.gsub!(/ë/, "\"e")
nname.gsub!(/ï/, "\"i")
nname.gsub!(/ö/, "\"o")
nname.gsub!(/ü/, "\"u")
nname.gsub!(/ñ/, "~n")
nname.gsub!(/#{86.chr}/, "U")
nname.gsub!(/#{151.chr}/, " ")
nname.gsub!(/#{243.chr}/, "L")
nname.gsub!(/#{247.chr}/, "S")
end
# shit that gets through...
nname.gsub!('#{146.chr}', "")
end
nname.gsub!(/Ä/, "A"); nname.gsub!(/ä/, "a")
nname.gsub!(/Á/, "A"); nname.gsub!(/á/, "a")
nname.gsub!(/À/, "A"); nname.gsub!(/à/, "a")
nname.gsub!(/Â/, "A"); nname.gsub!(/â/, "a")
nname.gsub!(/Ë/, "E"); nname.gsub!(/ë/, "e")
nname.gsub!(/É/, "E"); nname.gsub!(/é/, "e")
nname.gsub!(/È/, "E"); nname.gsub!(/è/, "e")
nname.gsub!(/Ê/, "E"); nname.gsub!(/ê/, "e")
nname.gsub!(/Ï/, "I"); nname.gsub!(/ï/, "i")
nname.gsub!(/Í/, "I"); nname.gsub!(/í/, "i")
nname.gsub!(/Ì/, "I"); nname.gsub!(/ì/, "i")
nname.gsub!(/Î/, "I"); nname.gsub!(/î/, "i")
nname.gsub!(/Ö/, "O"); nname.gsub!(/ö/, "o")
nname.gsub!(/Ó/, "O"); nname.gsub!(/ó/, "o")
nname.gsub!(/Ò/, "O"); nname.gsub!(/ò/, "o")
nname.gsub!(/Ô/, "O"); nname.gsub!(/ô/, "o")
nname.gsub!(/Ú/, "U"); nname.gsub!(/ú/, "u")
nname.gsub!(/Ü/, "U"); nname.gsub!(/ü/, "u")
nname.gsub!(/Ù/, "U"); nname.gsub!(/ù/, "u")
nname.gsub!(/Û/, "U"); nname.gsub!(/û/, "u")
nname.gsub!(/§/, "S"); nname.gsub!(/£/, "L")
nname.gsub!(/ß/, "ss")
### nname.gsub!('#{160.chr}', "")
### if nname.match(/([#{128.chr}-#{255.chr}])/)
### puts "replacing #{$1] with '*' in '#{nname}'"
### nname.tr!('#{128.chr}-#{255.chr}', "*") # catch all the other stupid shit
### end
nname.sub!(/\s*$/, "")
nname.sub!(/^[\s\.-]*/, "")
puts "nname: #{nname}" if Debuglevel > 1
newname = nname[0...@maxfilelength] # all the recodings could have made newname = nname[0...@maxfilelength] # all the recodings could have made
# the filename too long # the filename too long
count = 1 count = 1
@ -163,23 +99,26 @@ def save_file(dir, name, data)
begin begin
case data.class.to_s case data.class.to_s
when "String" when "String"
puts " Moving #{data} to #{dir}/#{newname}"
if File.move(data, "#{dir}/#{newname}") if File.move(data, "#{dir}/#{newname}")
puts " Saving as: '#{newname}'" puts " Saved as: '#{newname}'"
else else
puts "couldn't rename tempfile" puts "couldn't rename tempfile"
return false return false
end end
when "Array" when "Array"
puts " Saving to #{dir}/#{newname}"
if file = File.new("#{dir}/#{newname}", "w", 0644) if file = File.new("#{dir}/#{newname}", "w", 0644)
puts " Saving as: '#{newname}'" puts " Saved as: '#{newname}'"
data.collect{|i| file.print "#{i}"} data.collect{|i| file.print "#{i}"}
else else
puts "couldn't open file for writeing" puts "couldn't open file for writeing"
return false return false
end end
when "Tempfile" when "Tempfile"
puts " Moving #{data} to #{dir}/#{newname}"
if File.move(data.path, "#{dir}/#{newname}") if File.move(data.path, "#{dir}/#{newname}")
puts " Saving as: '#{newname}'" puts " Saved as: '#{newname}'"
else else
puts "couldn't rename tempfile #{data.path}" puts "couldn't rename tempfile #{data.path}"
return false return false
@ -201,6 +140,92 @@ def save_file(dir, name, data)
return true return true
end end
def filename_sanitize(nname)
# XXX I really have to find something a bit more elegant for this
# XXX this is a mac problem, should be configurable!!!!
group = @config.keys[0]
if @config[group].has_key?("ASCIIFILENAMES") && @config[group]["ASCIIFILENAMES"]
require 'iconv'
begin
# hmmm, misschien is de begin waarde wel gewoon geen UTF-8, maar 8-bit ASCII
# wat de mac filesystems nog slechter maakt overigens
nname = Iconv.conv("ASCII//TRANSLIT", "UTF-8", nname)
rescue Iconv::IllegalSequence
begin
nname = Iconv.conv("ASCII//TRANSLIT", "ISO-8859-1", nname)
rescue Iconv::IllegalSequence
# If it gets here, the original encoding is not UTF-8 of ISO-8859-1
# so the following probably won't help
puts "Iconv couldn't handle: '#{nname}'"
nname.gsub!(/\\\d+/, "#")
nname.gsub!(/ä/, "\"a")
nname.gsub!(/ë/, "\"e")
nname.gsub!(/ï/, "\"i")
nname.gsub!(/ö/, "\"o")
nname.gsub!(/ü/, "\"u")
nname.gsub!(/ñ/, "~n")
nname.gsub!(/#{86.chr}/, "U")
nname.gsub!(/#{151.chr}/, " ")
nname.gsub!(/#{161.chr}/, "i")
nname.gsub!(/#{164.chr}/, "")
nname.gsub!(/#{171.chr}/, " ")
nname.gsub!(/#{180.chr}/, " ")
nname.gsub!(/#{183.chr}/, " ")
nname.gsub!(/#{187.chr}/, " ")
nname.gsub!(/#{227.chr}/, "~a")
nname.gsub!(/#{229.chr}/, "a")
nname.gsub!(/#{231.chr}/, "c")
nname.gsub!(/#{243.chr}/, "o")
nname.gsub!(/#{247.chr}/, "S")
end
end
# shit that gets through...
nname.gsub!('#{146.chr}', "")
end
# I think the new iconv handling should fix this
# and it really doesn't belong on non apple machines
# nname.gsub!(/©/, "(c)")
# nname.gsub!(/®/, "(r)")
# nname.gsub!(/Ä/, "A"); nname.gsub!(/ä/, "a")
# nname.gsub!(/Á/, "A"); nname.gsub!(/á/, "a")
# nname.gsub!(/À/, "A"); nname.gsub!(/à/, "a")
# nname.gsub!(/Â/, "A"); nname.gsub!(/â/, "a")
#
# nname.gsub!(/Ë/, "E"); nname.gsub!(/ë/, "e")
# nname.gsub!(/É/, "E"); nname.gsub!(/é/, "e")
# nname.gsub!(/È/, "E"); nname.gsub!(/è/, "e")
# nname.gsub!(/Ê/, "E"); nname.gsub!(/ê/, "e")
#
# nname.gsub!(/Ï/, "I"); nname.gsub!(/ï/, "i")
# nname.gsub!(/Í/, "I"); nname.gsub!(/í/, "i")
# nname.gsub!(/Ì/, "I"); nname.gsub!(/ì/, "i")
# nname.gsub!(/Î/, "I"); nname.gsub!(/î/, "i")
#
# nname.gsub!(/Ö/, "O"); nname.gsub!(/ö/, "o")
# nname.gsub!(/Ó/, "O"); nname.gsub!(/ó/, "o")
# nname.gsub!(/Ò/, "O"); nname.gsub!(/ò/, "o")
# nname.gsub!(/Ô/, "O"); nname.gsub!(/ô/, "o")
#
# nname.gsub!(/Ú/, "U"); nname.gsub!(/ú/, "u")
# nname.gsub!(/Ü/, "U"); nname.gsub!(/ü/, "u")
# nname.gsub!(/Ù/, "U"); nname.gsub!(/ù/, "u")
# nname.gsub!(/Û/, "U"); nname.gsub!(/û/, "u")
#
# nname.gsub!(/§/, "S"); nname.gsub!(/£/, "L")
# nname.gsub!(/ß/, "ss")
# dangerous chars
nname = nname.gsub(/\//, "-")
nname.gsub!(/>/, "")
nname.gsub!(/</, "")
# just annoying
nname.sub!(/\s*$/, "")
nname.sub!(/^[\s\.-]*/, "")
puts "nname: #{nname}" if Debuglevel > 1
return nname
end
def parse_options(options) def parse_options(options)
begin begin
opts = GetoptLong.new( opts = GetoptLong.new(
@ -741,6 +766,7 @@ def main
for subj in @articles.get_group_subjects.sort{|a, b| ward_sort(a, b)} for subj in @articles.get_group_subjects.sort{|a, b| ward_sort(a, b)}
puts "#{subj}" if Debuglevel > 2 puts "#{subj}" if Debuglevel > 2
poster = @articles.get_group_poster(subj) poster = @articles.get_group_poster(subj)
postdate = @articles.get_group_date(subj)
# explicitly mark as read # explicitly mark as read
if @config[group].has_key?("-MR") and subj =~ /#{@config[group]["-MR"]}/ if @config[group].has_key?("-MR") and subj =~ /#{@config[group]["-MR"]}/
puts "Marking '#{subj}' as read" puts "Marking '#{subj}' as read"
@ -750,6 +776,11 @@ def main
puts "Marking poster '#{poster}' as read (subj: '#{subj}')" puts "Marking poster '#{poster}' as read (subj: '#{subj}')"
subjcount += 1 subjcount += 1
_mark_read(subj) _mark_read(subj)
# mark read old (Time.now - MRO days ).strf(YYYYMMDD).to_i ofzo
elsif @config[group].has_key?("-MRO") and postdate < (Time.now-(@config[group]["-MRO"].to_i*24*60*60)).strftime('%Y%m%d').to_i
puts "Marking date '#{postdate}' as read (subj: '#{subj}')"
subjcount += 1
_mark_read(subj)
# get the juicy bits # get the juicy bits
elsif !(@config[group].has_key?("-X") and subj =~ /#{@config[group]["-X"]}/) and elsif !(@config[group].has_key?("-X") and subj =~ /#{@config[group]["-X"]}/) and
subj =~ /#{@config[group]["-I"]}/ subj =~ /#{@config[group]["-I"]}/