406 lines
11 KiB
Ruby
Executable file
406 lines
11 KiB
Ruby
Executable file
#!/usr/local/bin/ruby -w
|
|
|
|
# $Dwarf: ripnews.rb,v 1.44 2003/04/19 11:10:32 ward Exp $
|
|
# $Source$
|
|
|
|
require 'date'
|
|
require 'getoptlong'
|
|
require 'news/article'
|
|
require 'news/newsrc'
|
|
require 'tempfile'
|
|
|
|
|
|
###########################################################################
|
|
|
|
Debuglevel = 0
|
|
|
|
def save_file(dir, name, data)
|
|
print "savename: #{name}\n" if Debuglevel > 1
|
|
nname = name.gsub(/\//, "-")
|
|
nname.sub!(/\s*$/, "")
|
|
nname.sub!(/^[\s\.]*/, "")
|
|
print "nname: #{nname}\n" if Debuglevel > 1
|
|
newname = nname
|
|
count = 1
|
|
d = Date.today
|
|
date = "#{d.year}#{d.month}#{d.mday}"
|
|
while FileTest.exists?("#{dir}/#{newname}")
|
|
newname = "#{nname}-<#{date}.#{count}>"
|
|
count += 1
|
|
end
|
|
print "name: #{newname}\n" if Debuglevel > 1
|
|
|
|
case data.type.to_s
|
|
when "String"
|
|
begin
|
|
if File.rename(data, "#{dir}/#{newname}")
|
|
print " Saving as: '#{newname}'\n"
|
|
else
|
|
print "couldn't rename tempfile\n"
|
|
return false
|
|
end
|
|
rescue Errno::ENOENT
|
|
print "Caught Errno::ENOENT (save_file)\n"
|
|
print "Error: #{$!}\n"
|
|
print "What the #@$$ happended?\n"
|
|
return false
|
|
end
|
|
when "Array"
|
|
if file = File.new("#{dir}/#{newname}", "w", "0644")
|
|
print " Saving as: '#{newname}'\n"
|
|
data.collect{|i| file.print "#{i}"}
|
|
else
|
|
print "couldn't open file for writeing\n"
|
|
return false
|
|
end
|
|
when "Tempfile"
|
|
begin
|
|
if File.rename(data.path, "#{dir}/#{newname}")
|
|
print " Saving as: '#{newname}'\n"
|
|
else
|
|
print "couldn't rename tempfile\n"
|
|
return false
|
|
end
|
|
rescue Errno::ENOENT
|
|
print "Caught Errno::ENOENT (save_file)\n"
|
|
print "Error: #{$!}\n"
|
|
print "What the #@$$ happended?\n"
|
|
return false
|
|
end
|
|
else
|
|
print "EEEEPS Can't save data of type: #{data.type.to_s}\n"
|
|
return false
|
|
end
|
|
return true
|
|
end
|
|
|
|
def parse_options(options)
|
|
begin
|
|
opts = GetoptLong.new(
|
|
[ "-I", "--include", GetoptLong::REQUIRED_ARGUMENT ],
|
|
[ "-c", "--configfile", GetoptLong::REQUIRED_ARGUMENT ],
|
|
[ "-L", "--longname", GetoptLong::NO_ARGUMENT ],
|
|
[ "-C", "--combinedname", GetoptLong::NO_ARGUMENT ],
|
|
[ "-M", "--multipart", GetoptLong::NO_ARGUMENT ],
|
|
[ "-S", "--singlepart", GetoptLong::NO_ARGUMENT ],
|
|
[ "-T", "--test", GetoptLong::NO_ARGUMENT ],
|
|
[ "-X", "--exclude", GetoptLong::REQUIRED_ARGUMENT ]
|
|
)
|
|
opts.quiet=true
|
|
|
|
opts.each do |opt, arg|
|
|
options[opt] = arg
|
|
end
|
|
rescue GetoptLong::InvalidOption
|
|
print "#{$!}\n"
|
|
usage
|
|
end
|
|
|
|
return options
|
|
end
|
|
|
|
def usage
|
|
print "\nUsage:\n\n"
|
|
print "ripnews.rb [-I <pattern>] [-c <file>] [-L] [-C] [-M] [-S] [-T] [-X <pattern>]\n\n"
|
|
print "-I <pattern> specify an include pattern\n"
|
|
print "-c <file> specify an alternate configfile\n"
|
|
print "-L use subject as filename\n"
|
|
print "-C use combined filenames\n"
|
|
print "-M get multipart articles\n"
|
|
print "-S get singlepart articles\n"
|
|
print "-T test mode, don't update newsrc file\n"
|
|
print "-X <pattern> specify an exclude pattern\n"
|
|
exit
|
|
end
|
|
|
|
def parse_config(default = {})
|
|
file = File.new("#{default[\"-c\"]}")
|
|
lines = file.readlines
|
|
|
|
i = 0
|
|
group = ""
|
|
grouparr = []
|
|
@config = {}
|
|
|
|
lines.collect!{|x|
|
|
x.sub!(/^\s*/, "")
|
|
x.sub!(/\#.*$/, "")
|
|
x.sub!(/\s*$/, "")
|
|
x.chomp
|
|
}
|
|
while i < lines.length
|
|
line = lines[i]
|
|
while line.sub!(/\s*\\$/, "") != nil
|
|
line << lines[i+1]
|
|
i += 1
|
|
end
|
|
line.sub!(/\s*$/, "")
|
|
i += 1
|
|
if line =~ /^OPT_(.*?)=(.*)/
|
|
line = "-#{$1}=#{$2}"
|
|
end
|
|
print "#{i}: #{line}\n" if Debuglevel > 1
|
|
if line =~ /(.*?)\s*\+=\s*(.*)/
|
|
if group == ""
|
|
if default.has_key?($1)
|
|
default[$1] << $2
|
|
else
|
|
default[$1] = $2
|
|
end
|
|
else
|
|
grouparr.collect{|g|
|
|
if @config[g].has_key?($1)
|
|
@config[g][$1] << $2
|
|
elsif default.has_key?($1)
|
|
@config[g][$1] = default[$1] + $2
|
|
else
|
|
@config[g][$1] = $2
|
|
end
|
|
}
|
|
end
|
|
elsif line =~ /(.*?)\s*=\s*(.*)/
|
|
if group == ""
|
|
default[$1] = $2
|
|
else
|
|
grouparr.collect{|g|
|
|
@config[g][$1] = $2
|
|
}
|
|
end
|
|
elsif line =~ /(.*?)\s*\{/
|
|
group = $1
|
|
grouparr = group.split('|')
|
|
grouparr.collect{|g|
|
|
@config[g] = {} unless @config.has_key?(g)
|
|
}
|
|
elsif line =~ /^}$/
|
|
default.each_key{|x|
|
|
grouparr.collect{|g|
|
|
@config[g][x] = default[x] unless @config[g].has_key?(x)
|
|
}
|
|
}
|
|
group = ""
|
|
grouparr = []
|
|
elsif line =~ /^$/
|
|
next
|
|
else
|
|
print "Error parsing config on line: #{i}\n"
|
|
exit
|
|
end
|
|
end
|
|
|
|
if group != ""
|
|
print "Error parsing config: group not terminated on line #{i}\n"
|
|
exit
|
|
end
|
|
|
|
if Debuglevel > 2
|
|
@config.each_key{|x|
|
|
print "Group: #{x}\n"
|
|
@config[x].each_key{|y|
|
|
print "Key: '#{y}' => Value: '#{@config[x][y]}'\n"
|
|
}
|
|
}
|
|
end
|
|
return true
|
|
end
|
|
|
|
def check_config
|
|
@config.each_key {|i|
|
|
unless @config[i].has_key?("-I")
|
|
print "No inclusions given for group #{i}. Won't match anything.\n"
|
|
exit
|
|
end
|
|
@config[i]["DATADIR"] ="." unless @config[i].has_key?("DATADIR")
|
|
@config[i]["PERMISSION"] = "0755" unless @config[i].has_key?("PERMISSION")
|
|
if @config[i].has_key?("EXTENSIONS")
|
|
@config[i]["-S"] = @config[i]["EXTENSIONS"]
|
|
@config[i]["-M"] = @config[i]["EXTENSIONS"]
|
|
end
|
|
@config[i]["-M"] = "(?!.*)" if @config[i].has_key?("-S") and ! @config[i].has_key?("-M")
|
|
@config[i]["-S"] = "(?!.*)" if @config[i].has_key?("-M") and ! @config[i].has_key?("-S")
|
|
}
|
|
end
|
|
|
|
def get_single(subj, group)
|
|
print "Fetching singlepart article: #{subj}\n"
|
|
body = @articles.get_group_body(subj)
|
|
if @articles.is_uuencoded(body)
|
|
mode, filename, body = @articles.uudecode(body)
|
|
elsif @articles.is_yencoded(body)
|
|
mode, filename, body = @articles.ydecode(body)
|
|
else
|
|
print " Unknown encoding (not UU, not yEnc), skipping...\n"
|
|
return false
|
|
end
|
|
return false unless check_ext(group, filename, "s")
|
|
return mode, filename, body
|
|
end
|
|
|
|
def get_multi(subj, group)
|
|
print "Fetching multipart article: #{subj}\n"
|
|
if @config[group]["TEMPDIR"] == nil or @config[group]["TEMPDIR"] == ""
|
|
body = @articles.get_group_body(subj)
|
|
if @articles.is_uuencoded(body)
|
|
mode, filename, body = @articles.uudecode(body)
|
|
elsif @articles.is_yencoded(body)
|
|
mode, filename, body = @articles.ydecode(body)
|
|
else
|
|
print " Unknown encoding (not UU, not yEnc), skipping...\n"
|
|
return false
|
|
end
|
|
return false unless check_ext(group, filename, "m")
|
|
return mode, filename, body
|
|
else
|
|
body = @articles.get_group_body_first(subj)
|
|
if @articles.is_uuencoded(body) or @articles.is_yencoded(body)
|
|
file = Tempfile.new("riptmp", @config[group]["TEMPDIR"])
|
|
body.collect{|x| file.print "#{x}\n"}
|
|
# hier moet een extensie check!!!
|
|
return false unless @articles.get_group_body_rest(subj, file)
|
|
fileout = Tempfile.new("riptmp", @config[group]["TEMPDIR"])
|
|
if @articles.is_uuencoded(body)
|
|
mode, filename, body = @articles.uudecode(file, fileout)
|
|
elsif @articles.is_yencoded(body)
|
|
mode, filename, body = @articles.ydecode(file, fileout)
|
|
end
|
|
# in plaats van hier
|
|
return false unless check_ext(group, filename, "m")
|
|
body = fileout.path
|
|
bodybase = body.sub(/\/[^\/]*$/, "")
|
|
File.rename(body, "#{bodybase}/ripnewsdecode")
|
|
body = "#{bodybase}/ripnewsdecode"
|
|
file.close
|
|
fileout.close(false)
|
|
return mode, filename, body
|
|
else
|
|
print " Unknown encoding (not UU, not yEnc), skipping...\n"
|
|
return false
|
|
end
|
|
end
|
|
end
|
|
|
|
def output_data(subject, mode, filename="", body="")
|
|
group = @articles.get_groupname
|
|
print " mode: #{mode}\n" if Debuglevel > 0
|
|
print " Filename: '#{filename}'\n" if Debuglevel > 0
|
|
|
|
# de-crap subject...
|
|
sub = subject.sub(/\s*$/, "") # strip trailing spaces
|
|
sub.sub!(/^[\s\.!-]*/, "") # strip leading spaces, dots, exclamation points and dashes
|
|
|
|
# decide on a filename
|
|
if @config[group].has_key?("-L") and @config[group]["-L"]
|
|
print "longname\n" if Debuglevel > 1
|
|
outfile = sub[0...@maxfilelength]
|
|
elsif @config[group].has_key?("-C") and @config[group]["-C"]
|
|
print "combinedname\n" if Debuglevel > 1
|
|
outfile = sub[0...@maxfilelength-filename.length-3]
|
|
outfile = "#{outfile} [#{filename}]"
|
|
if outfile.length > @maxfilelength
|
|
outfile = filename[0...@maxfilelength]
|
|
end
|
|
else
|
|
print "shortname\n" if Debuglevel > 1
|
|
outfile = filename[0...@maxfilelength]
|
|
end
|
|
|
|
# do the actual saving
|
|
if save_file("#{@config[group]["DATADIR"]}/#{group}", outfile, body)
|
|
@articles.group_update_newsrc(subject)
|
|
@articles.save_newsrc unless @config[group].has_key?("-T") and @config[group]["-T"]
|
|
end
|
|
end
|
|
|
|
def check_ext(group, filename, mode)
|
|
case mode
|
|
when "s"
|
|
return @config[group].has_key?("-S") ? ( filename =~ /\.(#{@config[group]["-S"]})$/ ) : true
|
|
when "m"
|
|
return @config[group].has_key?("-M") ? ( filename =~ /\.(#{@config[group]["-M"]})$/ ) : true
|
|
else
|
|
print "Illegal mode \"#{mode}\" in check_ext\n"
|
|
exit
|
|
end
|
|
end
|
|
|
|
def get_max_file_length(tempdir=".")
|
|
name = "a"*500
|
|
begin
|
|
file = File.new("#{tempdir}/#{name}", "w", "0644").close
|
|
File.delete("#{tempdir}/#{name}")
|
|
rescue Errno::ENAMETOOLONG
|
|
name = name[0...-1]
|
|
retry
|
|
end
|
|
# this is how many characters are still likely to be appended
|
|
# is the filename already exists '-<#{date}.#{count}>' in save_file
|
|
# this could be brought back to 5 '-<#{count}>' ...
|
|
return name.length - 14
|
|
end
|
|
|
|
#############################################################################################
|
|
|
|
$stdout.sync=true # line buffered output
|
|
defaults = {'-c' => "#{ENV['HOME']}/.ripnewsrc"}
|
|
defaults = parse_options(defaults)
|
|
parse_config(defaults)
|
|
check_config
|
|
|
|
@maxfilelength = get_max_file_length(@config[@config.keys[0]]["TEMPDIR"])
|
|
|
|
print "$Id$\n"
|
|
|
|
if Debuglevel > 2
|
|
@config.each_key{|i|
|
|
print "Group: #{i}\n"
|
|
@config[i].each_key{|j|
|
|
print "Opt: #{j} val: #{@config[i][j]}\n"
|
|
}
|
|
}
|
|
end
|
|
|
|
for group in @config.keys.sort
|
|
print "Getting articles for #{group}\n"
|
|
@articles = Article.new(@config[group]["NNTPSERVER"], group, @config[group]["NEWSRCNAME"])
|
|
# begin
|
|
@articles.get_articles(@config[group]["CACHEDIR"])
|
|
# rescue Article::
|
|
# print "Caught something: #{$!}\n"
|
|
# @articles.quit
|
|
# next
|
|
# end
|
|
|
|
unless FileTest.directory?("#{@config[group]["DATADIR"]}/#{group}") or
|
|
Dir.mkdir("#{@config[group]["DATADIR"]}/#{group}", @config[group]["PERMISSION"].oct)
|
|
print "eeeps, couldn't create dir\n"
|
|
exit
|
|
end
|
|
for i in @articles.get_group_subjects
|
|
print "#{i}\n" if Debuglevel > 2
|
|
if !(@config[group].has_key?("-X") and i =~ /#{@config[group]["-X"]}/) and
|
|
i =~ /#{@config[group]["-I"]}/
|
|
print "Match: #{i}\n" if Debuglevel > 0
|
|
if @articles.group_is_complete(i)
|
|
begin
|
|
if @articles.group_is_singlepart(i)
|
|
mode, filename, body = get_single(i, group)
|
|
elsif @articles.group_is_multipart(i)
|
|
mode, filename, body = get_multi(i, group)
|
|
end
|
|
if mode == false
|
|
print "Skipping article...\n"
|
|
else
|
|
output_data(i, mode, filename, body)
|
|
end
|
|
rescue Article::TempError, Article::PermError
|
|
print "#{$!}\n"
|
|
print "Skipping article...\n"
|
|
next
|
|
end
|
|
else
|
|
print " Not complete: #{i}\n"
|
|
end
|
|
end
|
|
end
|
|
@articles.quit
|
|
end
|