optimizations and new aging option
This commit is contained in:
parent
f18662baef
commit
f181edab96
4 changed files with 115 additions and 76 deletions
|
|
@ -135,6 +135,7 @@ OPT_MR=<pattern> Set "mark read" pattern. Ripnews will place
|
|||
again. Great for reducing memory usage when
|
||||
checking a group for the first time.
|
||||
OPT_MRF=<patter> Set "mark read from" pattern. Filters on posters.
|
||||
OPT_MRO=<days> Set "mark read old". Filters posts older that set days.
|
||||
OPT_MRR=<bool> Mark Remaining Read. If this is set to
|
||||
true and the article doesn't match an exclude or
|
||||
include pattern, the article will be
|
||||
|
|
|
|||
|
|
@ -26,7 +26,5 @@
|
|||
added to the main buffer
|
||||
[ ] running without a tempdir doesn't work at all
|
||||
[ ] don't drop connections to servers when switching groups
|
||||
[ ] keep connections to newsservers alive (don't timeout)
|
||||
[ ] a markread mechanism for every post that is not specifically
|
||||
included, this could be used to prevent excessively big caches
|
||||
[ ] limiter on running time. if running longer than X, abort
|
||||
[ ] implement article expiration
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
# $Source$
|
||||
|
||||
#
|
||||
# Copyright (c) 2002, 2003, 2004, 2005 Ward Wouts <ward@wouts.nl>
|
||||
# Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 Ward Wouts <ward@wouts.nl>
|
||||
#
|
||||
# Permission to use, copy, modify, and distribute this software for any
|
||||
# purpose with or without fee is hereby granted, provided that the above
|
||||
|
|
@ -236,7 +236,7 @@ def preselect(subject)
|
|||
end
|
||||
|
||||
def add(id, messid, date, from, subject, server)
|
||||
@messageinfo.push(Message.new(messid, id, date, from, server, subject))
|
||||
@messageinfo.push(Message.new(messid, id.to_i, date.to_i, from, server, subject))
|
||||
@grouped = false
|
||||
end
|
||||
|
||||
|
|
@ -681,6 +681,15 @@ def get_group_poster(subj)
|
|||
return @groups[subj]["messageinfo"][0][:from]
|
||||
end
|
||||
|
||||
def get_group_date(subj)
|
||||
group_subject_sort(subj)
|
||||
unless @groups[subj]["messageinfo"] != nil && @groups[subj]["messageinfo"][0][:date]
|
||||
p "ieks komt niet door lame check heen"
|
||||
return false
|
||||
end
|
||||
return @groups[subj]["messageinfo"][0][:date]
|
||||
end
|
||||
|
||||
def group_is_complete(subj)
|
||||
group_subjects unless @grouped
|
||||
#print "Subject: #{subj}\n"
|
||||
|
|
@ -823,7 +832,7 @@ puts "#{Time.now} Reading & scrubbing caches"
|
|||
puts " #{Time.now} Reading cache for #{server}"
|
||||
excludes[server] = {}
|
||||
@connections[server]["skip_ids"].elements.collect!{|x| excludes[server][x]=true}
|
||||
if FileTest.directory?( cachedir) and FileTest.file?( "#{filename}.#{server}" ) and FileTest.readable?( "#{filename}.#{server}" )
|
||||
if FileTest.directory?(cachedir) and FileTest.file?( "#{filename}.#{server}" ) and FileTest.readable?( "#{filename}.#{server}" )
|
||||
outfile = File.new("#{filename}.#{server}.new", "w") or puts "Couldn't open cachefile for writing"
|
||||
File.new( "#{filename}.#{server}" ).each{ |line|
|
||||
line =~ regexp
|
||||
|
|
@ -833,7 +842,7 @@ puts " #{Time.now} Reading cache for #{server}"
|
|||
from = $4
|
||||
subject = $5
|
||||
if first <= id_i and id_i <= last
|
||||
unless excludes[server].has_key?(id_i)
|
||||
if ! excludes[server].has_key?(id_i)
|
||||
outfile.puts(line)
|
||||
if preselect(subject)
|
||||
add(id_i, messid, date, from, subject, server)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
# $Source$
|
||||
|
||||
#
|
||||
# Copyright (c) 2002, 2003, 2004, 2005 Ward Wouts <ward@wouts.nl>
|
||||
# Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 Ward Wouts <ward@wouts.nl>
|
||||
#
|
||||
# Permission to use, copy, modify, and distribute this software for any
|
||||
# purpose with or without fee is hereby granted, provided that the above
|
||||
|
|
@ -83,72 +83,8 @@ Debuglevel = 0
|
|||
def save_file(dir, name, data)
|
||||
puts "savename: #{name}" if Debuglevel > 1
|
||||
|
||||
nname = name.gsub(/\//, "-")
|
||||
nname.gsub!(/>/, "")
|
||||
nname.gsub!(/</, "")
|
||||
nname.gsub!(/©/, "(c)")
|
||||
nname.gsub!(/®/, "(r)")
|
||||
# XXX I really have to find something a bit more elegant for this
|
||||
# XXX this is a mac problem, should be configurable!!!!
|
||||
group = @config.keys[0]
|
||||
if @config[group].has_key?("ASCIIFILENAMES") && @config[group]["ASCIIFILENAMES"]
|
||||
require 'iconv'
|
||||
begin
|
||||
nname = Iconv.conv("ASCII//TRANSLIT", "UTF-8", nname)
|
||||
rescue Iconv::IllegalSequence
|
||||
puts "Iconv couldn't handle: '#{nname}'"
|
||||
nname.gsub!(/\\\d+/, "#")
|
||||
nname.gsub!(/ä/, "\"a")
|
||||
nname.gsub!(/ë/, "\"e")
|
||||
nname.gsub!(/ï/, "\"i")
|
||||
nname.gsub!(/ö/, "\"o")
|
||||
nname.gsub!(/ü/, "\"u")
|
||||
nname.gsub!(/ñ/, "~n")
|
||||
nname.gsub!(/#{86.chr}/, "U")
|
||||
nname.gsub!(/#{151.chr}/, " ")
|
||||
nname.gsub!(/#{243.chr}/, "L")
|
||||
nname.gsub!(/#{247.chr}/, "S")
|
||||
end
|
||||
# shit that gets through...
|
||||
nname.gsub!('#{146.chr}', "")
|
||||
end
|
||||
nname.gsub!(/Ä/, "A"); nname.gsub!(/ä/, "a")
|
||||
nname.gsub!(/Á/, "A"); nname.gsub!(/á/, "a")
|
||||
nname.gsub!(/À/, "A"); nname.gsub!(/à/, "a")
|
||||
nname.gsub!(/Â/, "A"); nname.gsub!(/â/, "a")
|
||||
nname = filename_sanitize(name)
|
||||
|
||||
nname.gsub!(/Ë/, "E"); nname.gsub!(/ë/, "e")
|
||||
nname.gsub!(/É/, "E"); nname.gsub!(/é/, "e")
|
||||
nname.gsub!(/È/, "E"); nname.gsub!(/è/, "e")
|
||||
nname.gsub!(/Ê/, "E"); nname.gsub!(/ê/, "e")
|
||||
|
||||
nname.gsub!(/Ï/, "I"); nname.gsub!(/ï/, "i")
|
||||
nname.gsub!(/Í/, "I"); nname.gsub!(/í/, "i")
|
||||
nname.gsub!(/Ì/, "I"); nname.gsub!(/ì/, "i")
|
||||
nname.gsub!(/Î/, "I"); nname.gsub!(/î/, "i")
|
||||
|
||||
nname.gsub!(/Ö/, "O"); nname.gsub!(/ö/, "o")
|
||||
nname.gsub!(/Ó/, "O"); nname.gsub!(/ó/, "o")
|
||||
nname.gsub!(/Ò/, "O"); nname.gsub!(/ò/, "o")
|
||||
nname.gsub!(/Ô/, "O"); nname.gsub!(/ô/, "o")
|
||||
|
||||
nname.gsub!(/Ú/, "U"); nname.gsub!(/ú/, "u")
|
||||
nname.gsub!(/Ü/, "U"); nname.gsub!(/ü/, "u")
|
||||
nname.gsub!(/Ù/, "U"); nname.gsub!(/ù/, "u")
|
||||
nname.gsub!(/Û/, "U"); nname.gsub!(/û/, "u")
|
||||
|
||||
nname.gsub!(/§/, "S"); nname.gsub!(/£/, "L")
|
||||
nname.gsub!(/ß/, "ss")
|
||||
|
||||
### nname.gsub!('#{160.chr}', "")
|
||||
### if nname.match(/([#{128.chr}-#{255.chr}])/)
|
||||
### puts "replacing #{$1] with '*' in '#{nname}'"
|
||||
### nname.tr!('#{128.chr}-#{255.chr}', "*") # catch all the other stupid shit
|
||||
### end
|
||||
|
||||
nname.sub!(/\s*$/, "")
|
||||
nname.sub!(/^[\s\.-]*/, "")
|
||||
puts "nname: #{nname}" if Debuglevel > 1
|
||||
newname = nname[0...@maxfilelength] # all the recodings could have made
|
||||
# the filename too long
|
||||
count = 1
|
||||
|
|
@ -163,23 +99,26 @@ def save_file(dir, name, data)
|
|||
begin
|
||||
case data.class.to_s
|
||||
when "String"
|
||||
puts " Moving #{data} to #{dir}/#{newname}"
|
||||
if File.move(data, "#{dir}/#{newname}")
|
||||
puts " Saving as: '#{newname}'"
|
||||
puts " Saved as: '#{newname}'"
|
||||
else
|
||||
puts "couldn't rename tempfile"
|
||||
return false
|
||||
end
|
||||
when "Array"
|
||||
puts " Saving to #{dir}/#{newname}"
|
||||
if file = File.new("#{dir}/#{newname}", "w", 0644)
|
||||
puts " Saving as: '#{newname}'"
|
||||
puts " Saved as: '#{newname}'"
|
||||
data.collect{|i| file.print "#{i}"}
|
||||
else
|
||||
puts "couldn't open file for writeing"
|
||||
return false
|
||||
end
|
||||
when "Tempfile"
|
||||
puts " Moving #{data} to #{dir}/#{newname}"
|
||||
if File.move(data.path, "#{dir}/#{newname}")
|
||||
puts " Saving as: '#{newname}'"
|
||||
puts " Saved as: '#{newname}'"
|
||||
else
|
||||
puts "couldn't rename tempfile #{data.path}"
|
||||
return false
|
||||
|
|
@ -201,6 +140,92 @@ def save_file(dir, name, data)
|
|||
return true
|
||||
end
|
||||
|
||||
def filename_sanitize(nname)
|
||||
# XXX I really have to find something a bit more elegant for this
|
||||
# XXX this is a mac problem, should be configurable!!!!
|
||||
group = @config.keys[0]
|
||||
if @config[group].has_key?("ASCIIFILENAMES") && @config[group]["ASCIIFILENAMES"]
|
||||
require 'iconv'
|
||||
begin
|
||||
# hmmm, misschien is de begin waarde wel gewoon geen UTF-8, maar 8-bit ASCII
|
||||
# wat de mac filesystems nog slechter maakt overigens
|
||||
nname = Iconv.conv("ASCII//TRANSLIT", "UTF-8", nname)
|
||||
rescue Iconv::IllegalSequence
|
||||
begin
|
||||
nname = Iconv.conv("ASCII//TRANSLIT", "ISO-8859-1", nname)
|
||||
rescue Iconv::IllegalSequence
|
||||
# If it gets here, the original encoding is not UTF-8 of ISO-8859-1
|
||||
# so the following probably won't help
|
||||
puts "Iconv couldn't handle: '#{nname}'"
|
||||
nname.gsub!(/\\\d+/, "#")
|
||||
nname.gsub!(/ä/, "\"a")
|
||||
nname.gsub!(/ë/, "\"e")
|
||||
nname.gsub!(/ï/, "\"i")
|
||||
nname.gsub!(/ö/, "\"o")
|
||||
nname.gsub!(/ü/, "\"u")
|
||||
nname.gsub!(/ñ/, "~n")
|
||||
nname.gsub!(/#{86.chr}/, "U")
|
||||
nname.gsub!(/#{151.chr}/, " ")
|
||||
nname.gsub!(/#{161.chr}/, "i")
|
||||
nname.gsub!(/#{164.chr}/, "")
|
||||
nname.gsub!(/#{171.chr}/, " ")
|
||||
nname.gsub!(/#{180.chr}/, " ")
|
||||
nname.gsub!(/#{183.chr}/, " ")
|
||||
nname.gsub!(/#{187.chr}/, " ")
|
||||
nname.gsub!(/#{227.chr}/, "~a")
|
||||
nname.gsub!(/#{229.chr}/, "a")
|
||||
nname.gsub!(/#{231.chr}/, "c")
|
||||
nname.gsub!(/#{243.chr}/, "o")
|
||||
nname.gsub!(/#{247.chr}/, "S")
|
||||
end
|
||||
end
|
||||
# shit that gets through...
|
||||
nname.gsub!('#{146.chr}', "")
|
||||
end
|
||||
# I think the new iconv handling should fix this
|
||||
# and it really doesn't belong on non apple machines
|
||||
# nname.gsub!(/©/, "(c)")
|
||||
# nname.gsub!(/®/, "(r)")
|
||||
# nname.gsub!(/Ä/, "A"); nname.gsub!(/ä/, "a")
|
||||
# nname.gsub!(/Á/, "A"); nname.gsub!(/á/, "a")
|
||||
# nname.gsub!(/À/, "A"); nname.gsub!(/à/, "a")
|
||||
# nname.gsub!(/Â/, "A"); nname.gsub!(/â/, "a")
|
||||
#
|
||||
# nname.gsub!(/Ë/, "E"); nname.gsub!(/ë/, "e")
|
||||
# nname.gsub!(/É/, "E"); nname.gsub!(/é/, "e")
|
||||
# nname.gsub!(/È/, "E"); nname.gsub!(/è/, "e")
|
||||
# nname.gsub!(/Ê/, "E"); nname.gsub!(/ê/, "e")
|
||||
#
|
||||
# nname.gsub!(/Ï/, "I"); nname.gsub!(/ï/, "i")
|
||||
# nname.gsub!(/Í/, "I"); nname.gsub!(/í/, "i")
|
||||
# nname.gsub!(/Ì/, "I"); nname.gsub!(/ì/, "i")
|
||||
# nname.gsub!(/Î/, "I"); nname.gsub!(/î/, "i")
|
||||
#
|
||||
# nname.gsub!(/Ö/, "O"); nname.gsub!(/ö/, "o")
|
||||
# nname.gsub!(/Ó/, "O"); nname.gsub!(/ó/, "o")
|
||||
# nname.gsub!(/Ò/, "O"); nname.gsub!(/ò/, "o")
|
||||
# nname.gsub!(/Ô/, "O"); nname.gsub!(/ô/, "o")
|
||||
#
|
||||
# nname.gsub!(/Ú/, "U"); nname.gsub!(/ú/, "u")
|
||||
# nname.gsub!(/Ü/, "U"); nname.gsub!(/ü/, "u")
|
||||
# nname.gsub!(/Ù/, "U"); nname.gsub!(/ù/, "u")
|
||||
# nname.gsub!(/Û/, "U"); nname.gsub!(/û/, "u")
|
||||
#
|
||||
# nname.gsub!(/§/, "S"); nname.gsub!(/£/, "L")
|
||||
# nname.gsub!(/ß/, "ss")
|
||||
|
||||
# dangerous chars
|
||||
nname = nname.gsub(/\//, "-")
|
||||
nname.gsub!(/>/, "")
|
||||
nname.gsub!(/</, "")
|
||||
|
||||
# just annoying
|
||||
nname.sub!(/\s*$/, "")
|
||||
nname.sub!(/^[\s\.-]*/, "")
|
||||
puts "nname: #{nname}" if Debuglevel > 1
|
||||
return nname
|
||||
end
|
||||
|
||||
def parse_options(options)
|
||||
begin
|
||||
opts = GetoptLong.new(
|
||||
|
|
@ -741,6 +766,7 @@ def main
|
|||
for subj in @articles.get_group_subjects.sort{|a, b| ward_sort(a, b)}
|
||||
puts "#{subj}" if Debuglevel > 2
|
||||
poster = @articles.get_group_poster(subj)
|
||||
postdate = @articles.get_group_date(subj)
|
||||
# explicitly mark as read
|
||||
if @config[group].has_key?("-MR") and subj =~ /#{@config[group]["-MR"]}/
|
||||
puts "Marking '#{subj}' as read"
|
||||
|
|
@ -750,6 +776,11 @@ def main
|
|||
puts "Marking poster '#{poster}' as read (subj: '#{subj}')"
|
||||
subjcount += 1
|
||||
_mark_read(subj)
|
||||
# mark read old (Time.now - MRO days ).strf(YYYYMMDD).to_i ofzo
|
||||
elsif @config[group].has_key?("-MRO") and postdate < (Time.now-(@config[group]["-MRO"].to_i*24*60*60)).strftime('%Y%m%d').to_i
|
||||
puts "Marking date '#{postdate}' as read (subj: '#{subj}')"
|
||||
subjcount += 1
|
||||
_mark_read(subj)
|
||||
# get the juicy bits
|
||||
elsif !(@config[group].has_key?("-X") and subj =~ /#{@config[group]["-X"]}/) and
|
||||
subj =~ /#{@config[group]["-I"]}/
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue