First add of ripnews to cvs
This commit is contained in:
parent
bde9d157e2
commit
05e19814d6
3 changed files with 1750 additions and 0 deletions
519
trunk/ripnews/ripnews.rb
Executable file
519
trunk/ripnews/ripnews.rb
Executable file
|
|
@ -0,0 +1,519 @@
|
|||
#!/usr/local/bin/ruby
|
||||
|
||||
require 'net/nntp'
|
||||
require 'news/newsrc'
|
||||
require 'date'
|
||||
require 'getoptlong'
|
||||
|
||||
class Articles
|
||||
|
||||
Debuglevel = 1
|
||||
|
||||
def initialize(server)
|
||||
@ids = []
|
||||
@subjects = []
|
||||
@sorted = false
|
||||
@grouped = false
|
||||
@agroups = {}
|
||||
@nntp = Net::NNTP.new(server)
|
||||
end
|
||||
|
||||
def add(id, subject)
|
||||
@ids += [id]
|
||||
@subjects += [subject]
|
||||
@sorted = false
|
||||
@grouped = false
|
||||
end
|
||||
|
||||
def get_articles(group)
|
||||
resp, count, first, last,name = @nntp.group(group)
|
||||
for i in (first.to_i..last.to_i)
|
||||
begin
|
||||
@nntp.stat(i)
|
||||
resp, nr, messid, list = @nntp.head(i)
|
||||
for j in list
|
||||
if j =~ /Subject: (.*)/
|
||||
subj=$1
|
||||
end
|
||||
end
|
||||
add(messid, subj)
|
||||
rescue Net::NNTP::RuntimeError
|
||||
print "whoopsie couldn't stat #{i}\n" if Debuglevel > 0
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def get_groups
|
||||
group_subjects unless @grouped
|
||||
return @agroups
|
||||
end
|
||||
|
||||
def get_group_body(subj)
|
||||
result = []
|
||||
for i in @agroups[subj][1..@agroups[subj].length]
|
||||
resp, nr, id, list = @nntp.body(i)
|
||||
result = list
|
||||
end
|
||||
return result
|
||||
end
|
||||
|
||||
def get_group_body_first(subj)
|
||||
resp, nr, id, list = @nntp.body(@agroups[subj][1])
|
||||
print "getting article: #{subj}\n" if Debuglevel > 0
|
||||
print "article id: #{id}\n" if Debuglevel > 0
|
||||
return list
|
||||
end
|
||||
|
||||
def get_group_body_rest(subj, file=nil)
|
||||
result = []
|
||||
for i in @agroups[subj][2..@agroups[subj].length]
|
||||
print "getting article: #{i}\n" if Debuglevel > 0
|
||||
resp, nr, id, list = @nntp.body(i)
|
||||
if file
|
||||
for line in list
|
||||
file.print "#{line}\n"
|
||||
end
|
||||
else
|
||||
result += list
|
||||
end
|
||||
end
|
||||
return result
|
||||
end
|
||||
|
||||
def get_group_subjects
|
||||
group_subjects unless @grouped
|
||||
return @agroups.keys
|
||||
end
|
||||
|
||||
def group_complete(subj)
|
||||
group_subjects unless @grouped
|
||||
print "length: #{@agroups[subj].length} total: #{@agroups[subj][0].to_i}\n" if Debuglevel > 0
|
||||
if (@agroups[subj].length - 1 ) >= @agroups[subj][0].to_i
|
||||
return true
|
||||
else
|
||||
return false
|
||||
end
|
||||
end
|
||||
|
||||
def get_ids
|
||||
return @ids
|
||||
end
|
||||
|
||||
def get_subjects
|
||||
return @subjects
|
||||
end
|
||||
|
||||
def group_subjects
|
||||
@agroups = {}
|
||||
subject_sort unless @sorted
|
||||
prev_subj = ""
|
||||
for i in (0..@subjects.length)
|
||||
if @subjects[i] =~ /(.*)\((\d+)\/(\d+)\)(.*)/ || @subjects[i] =~ /(.*)\[(\d+)\/(\d+)\](.*)/
|
||||
j = "#{$1}#{$4}"
|
||||
number = $2
|
||||
total = $3
|
||||
else
|
||||
j = @subjects[i]
|
||||
number = 1
|
||||
total = 1
|
||||
end
|
||||
if j == prev_subj
|
||||
@agroups[j] += [ @ids[i] ]
|
||||
else
|
||||
unless number == 0
|
||||
prev_subj = j
|
||||
@agroups[j] = [ total, @ids[i] ]
|
||||
end
|
||||
end
|
||||
end
|
||||
@grouped = true
|
||||
end
|
||||
|
||||
def uudecode(data, outfile=nil)
|
||||
case data.type.to_s
|
||||
when "Array"
|
||||
print "Calling _uudecode_array\n" if Debuglevel>0
|
||||
mode, file, body = _uudecode_array(data)
|
||||
when "File"
|
||||
unless outfile
|
||||
print "uudecode: need outfile\n"
|
||||
exit
|
||||
end
|
||||
print "Calling _uudecode_file\n" if Debuglevel>0
|
||||
mode, file, body = _uudecode_file(data, outfile)
|
||||
end
|
||||
return mode, file, body
|
||||
end
|
||||
|
||||
def _uudecode_file(file, outfile)
|
||||
mode = 0600
|
||||
filename = "unknown"
|
||||
c = 0
|
||||
lines = file.pos
|
||||
percent = 0
|
||||
mark = lines/100
|
||||
file.pos=0
|
||||
|
||||
while (! file.eof)
|
||||
line = file.gets
|
||||
print "line: #{line}" if Debuglevel > 0
|
||||
if line =~ /^begin(.*)/
|
||||
m = $1
|
||||
print "beginning matched; rest: #{m}\n"
|
||||
if m =~ /^(\s+(\d+))?(\s+(.*?\S))?\s*\Z/
|
||||
mode = $2
|
||||
filename = $4
|
||||
print "found beginning\n" if Debuglevel > 0
|
||||
else
|
||||
print "mode, file set to defaults: #{m}\n"
|
||||
end
|
||||
break
|
||||
end
|
||||
end
|
||||
|
||||
print "not uuencoded!\n" if file.eof
|
||||
print "c: #{c} mark: #{mark} lines: #{lines}\n" if Debuglevel > 1
|
||||
|
||||
print "UUdecoding...\n"
|
||||
|
||||
while (! file.eof)
|
||||
if Debuglevel > 1
|
||||
c = file.pos
|
||||
if c > mark
|
||||
print "#{percent}%\n"
|
||||
print "c: #{c} mark: #{mark} lines: #{lines}\n" if Debuglevel > 1
|
||||
percent += 1
|
||||
mark = (lines/100)*(percent+1)
|
||||
end
|
||||
end
|
||||
line = file.gets
|
||||
print "line: #{line}" if Debuglevel > 1
|
||||
return mode, filename if line =~ /^end/
|
||||
next if line =~ /[a-z]/
|
||||
next if line == nil
|
||||
next unless ((((line[0] - 32) & 077) + 2) / 3).to_i ==
|
||||
(line.length/4).to_i
|
||||
outfile.print line.unpack("u")
|
||||
end
|
||||
|
||||
print "No \"end\" found!!!\n"
|
||||
return mode, file, outfile
|
||||
|
||||
end
|
||||
|
||||
# gaat volgens mij niet verder als er meerdere uuencoded blocks zijn...
|
||||
# zal dan meerdere keren aangeroepen moeten worden, grmbl...
|
||||
# tis getting a mess as we speak...
|
||||
# toch maar een keer aparte class van maken...
|
||||
def _uudecode_array(data)
|
||||
decode = []
|
||||
# begun = false
|
||||
mode = 0600
|
||||
file = "unknown"
|
||||
c = 0
|
||||
lines = data.length
|
||||
percent = 0
|
||||
mark = lines/100
|
||||
|
||||
i = 0
|
||||
print "data.length #{data.length}\n"
|
||||
while (i < data.length)
|
||||
print "i #{i}\n"
|
||||
if data[i] =~ /^begin(.*)/
|
||||
m = $1
|
||||
print "beginning matched; rest: #{m}\n"
|
||||
if m =~ /^(\s+(\d+))?(\s+(.*?\S))?\s*\Z/
|
||||
mode = $2
|
||||
file = $4
|
||||
print "found beginning\n" if Debuglevel > 0
|
||||
else
|
||||
print "mode, file set to defaults: #{m}\n"
|
||||
end
|
||||
break
|
||||
end
|
||||
i += 1
|
||||
end
|
||||
|
||||
unless (i < data.length)
|
||||
print "not uuencoded!\n"
|
||||
end
|
||||
|
||||
print "UUdecoding...\n"
|
||||
|
||||
while (i < data.length)
|
||||
if Debuglevel > 1
|
||||
if c > mark
|
||||
print "#{percent}%\n"
|
||||
print "c: #{c} mark: #{mark} lines: #{lines} i: #{i}\n" if Debuglevel > 1
|
||||
percent += 1
|
||||
mark = (lines/100)*(percent+1)
|
||||
end
|
||||
c += 1
|
||||
end
|
||||
line = data[i]
|
||||
i += 1
|
||||
return mode, file, decode if line =~ /^end/
|
||||
next if line =~ /[a-z]/
|
||||
next if line == nil
|
||||
next unless ((((line[0] - 32) & 077) + 2) / 3).to_i ==
|
||||
(line.length/4).to_i
|
||||
decode += line.unpack("u")
|
||||
end
|
||||
|
||||
print "No \"end\" found!!!\n"
|
||||
return mode, file, decode
|
||||
end
|
||||
|
||||
def uudecode_group(subj, file=nil, outfile=nil)
|
||||
group_subjects unless @grouped
|
||||
|
||||
body = get_group_body_first(subj)
|
||||
if body.to_s =~ /begin/
|
||||
print "uuencoded!\n" if Debuglevel > 0
|
||||
if (file and outfile)
|
||||
for i in body
|
||||
file.print "#{i}\n"
|
||||
end
|
||||
get_group_body_rest(subj, file)
|
||||
mode, filename, result = uudecode(file, outfile)
|
||||
else
|
||||
body += get_group_body_rest(subj)
|
||||
mode, filename, result = uudecode(body)
|
||||
end
|
||||
return mode, filename, result
|
||||
else
|
||||
print "Not uuencoded!\n" if Debuglevel > 0
|
||||
return false
|
||||
end
|
||||
end
|
||||
|
||||
###############################################################
|
||||
|
||||
def subject_sort
|
||||
sort_arr = []
|
||||
for i in (0..@subjects.length)
|
||||
sort_arr += ["#{@subjects[i]} #{@ids[i]}"]
|
||||
end
|
||||
sort_arr.sort!{|a,b| ward_sort(a, b)}
|
||||
@ids = []
|
||||
@subjects = []
|
||||
for i in sort_arr
|
||||
i =~ /^(.*) (<[^<]*>)$/ || i =~ /^(.*) \[<[^<]*>\]$/
|
||||
@ids += [$2]
|
||||
@subjects += [$1]
|
||||
end
|
||||
@sorted = true
|
||||
end
|
||||
|
||||
def ward_sort(a, b)
|
||||
a =~ /^(.*) (<[^<]*>)$/
|
||||
c = $1.to_s.split(/([0-9]+)/)
|
||||
b =~ /^(.*) (<[^<]*>)$/
|
||||
d = $1.to_s.split(/([0-9]+)/)
|
||||
|
||||
for x in c
|
||||
y = d.shift
|
||||
r = ((x.to_s =~ /^[0-9]+$/) && (y.to_s =~ /^[0-9]+$/)) ?
|
||||
(x.to_i <=> y.to_i) :
|
||||
(x.to_s <=> y.to_s)
|
||||
if r != 0
|
||||
return r
|
||||
end
|
||||
end
|
||||
return -1 if (d)
|
||||
return 0
|
||||
end
|
||||
|
||||
def quit
|
||||
@nntp.quit
|
||||
end
|
||||
|
||||
private :ward_sort
|
||||
|
||||
end
|
||||
|
||||
###########################################################################
|
||||
###########################################################################
|
||||
|
||||
Debuglevel = 1
|
||||
|
||||
def save_file(dir, name, data)
|
||||
print "savename: #{name}\n" if Debuglevel > 1
|
||||
nname = name.gsub(/\//, "-")
|
||||
print "nname: #{nname}\n" if Debuglevel > 1
|
||||
newname = nname
|
||||
count = 1
|
||||
d = Date.today
|
||||
date = "#{d.year}#{d.month}#{d.mday}"
|
||||
while FileTest.exists?("#{dir}/#{newname}")
|
||||
newname = "#{nname}-<#{date}.#{count}>"
|
||||
count += 1
|
||||
end
|
||||
print "name: #{newname}\n" if Debuglevel > 1
|
||||
|
||||
case data.type.to_s
|
||||
when "String"
|
||||
if File.rename(data, "#{dir}/#{newname}")
|
||||
print "Saving: #{newname}\n"
|
||||
else
|
||||
print "couldn't rename tempfile\n"
|
||||
end
|
||||
when "Array"
|
||||
if file = File.new("#{dir}/#{newname}", "w", "0644")
|
||||
print "Saving: #{newname}\n"
|
||||
for i in data
|
||||
file.print "#{i}"
|
||||
end
|
||||
else
|
||||
print "couldn't open file for writeing\n"
|
||||
end
|
||||
else
|
||||
print "EEEEPS\n"
|
||||
end
|
||||
end
|
||||
|
||||
def tmp_file(dir)
|
||||
name = "riptmp"
|
||||
print "tmpname: #{name}\n" if Debuglevel > 1
|
||||
nname = name.gsub(/\//, "-")
|
||||
print "nname: #{nname}\n" if Debuglevel > 1
|
||||
newname = nname
|
||||
count = 1
|
||||
|
||||
while FileTest.exists?("#{dir}/#{newname}")
|
||||
newname = "#{nname}-#{count}"
|
||||
count += 1
|
||||
end
|
||||
print "name: #{newname}\n" if Debuglevel > 1
|
||||
fullname ="#{dir}/#{newname}"
|
||||
if file = File.new(fullname, "w+", "0644")
|
||||
return file, fullname
|
||||
else
|
||||
print "couldn't open tempfile for writeing\n"
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
def parse_options
|
||||
options = {}
|
||||
|
||||
begin
|
||||
opts = GetoptLong.new(
|
||||
[ "-I", "--include", GetoptLong::REQUIRED_ARGUMENT ],
|
||||
[ "-c", "--configfile", GetoptLong::REQUIRED_ARGUMENT ],
|
||||
[ "-L", "--longname", GetoptLong::NO_ARGUMENT ],
|
||||
[ "-S", "--singlepart", GetoptLong::NO_ARGUMENT ],
|
||||
[ "-g", "--greedy", GetoptLong::NO_ARGUMENT ]
|
||||
)
|
||||
opts.quiet=true
|
||||
|
||||
opts.each do |opt, arg|
|
||||
options[opt] = arg
|
||||
end
|
||||
|
||||
rescue
|
||||
print "#{opts.error_message}\n"
|
||||
print "\nUsage:\n"
|
||||
exit
|
||||
end
|
||||
|
||||
return options
|
||||
end
|
||||
|
||||
def read_config(options)
|
||||
unless options.has_key?("-c")
|
||||
options["-c"]=".ripnewsrc"
|
||||
end
|
||||
begin
|
||||
config = IO.readlines("#{options[\"-c\"]}")
|
||||
for i in config
|
||||
if i =~ /^OPT_(.)=?(.*)$/
|
||||
options["-#{$1}"]=$2 unless
|
||||
options.has_key?("-#{$1}")
|
||||
else
|
||||
i =~ /([^=]*)=(.*)/
|
||||
options["#{$1}"]=$2
|
||||
end
|
||||
print "#{$1}=", options["#{$1}"], "\n" if Debuglevel > 1
|
||||
end
|
||||
rescue
|
||||
print "Coudn't open config file: #{options[\"-c\"]}\n"
|
||||
exit
|
||||
end
|
||||
return options
|
||||
end
|
||||
|
||||
def check_options(options)
|
||||
if (Debuglevel > 1)
|
||||
for i in options.keys
|
||||
print "Opt: #{i} Value: #{options[i]}\n"
|
||||
end
|
||||
end
|
||||
unless options.has_key?("-I")
|
||||
print "No inclusions given. Won't match anything.\n"
|
||||
exit
|
||||
end
|
||||
unless options.has_key?("DATADIR")
|
||||
options["DATADIR"] ="."
|
||||
end
|
||||
# exit # zeer tijdelijk voor snel testen
|
||||
end
|
||||
|
||||
options = {}
|
||||
options = parse_options
|
||||
options = read_config(options)
|
||||
check_options(options)
|
||||
|
||||
if Debuglevel > 1
|
||||
for i in options.keys
|
||||
print "Opt: #{i} val: #{options[i]}\n"
|
||||
end
|
||||
end
|
||||
|
||||
articles = Articles.new(options["NNTPSERVER"])
|
||||
articles.get_articles("alt.binaries.e-book.flood")
|
||||
|
||||
|
||||
for i in articles.get_group_subjects
|
||||
if i =~ /#{options["-I"]}/i
|
||||
print "Match: #{i}\n" if Debuglevel > 0
|
||||
if articles.group_complete(i)
|
||||
print "Complete: #{i}\n" if Debuglevel > 0
|
||||
if options.has_key?("TMPDIR")
|
||||
file, fullname = tmp_file(options["TMPDIR"])
|
||||
fileout, fullnameout = tmp_file(options["TMPDIR"])
|
||||
else
|
||||
file=nil
|
||||
fullname = ""
|
||||
fileout=nil
|
||||
fullnameout = ""
|
||||
end
|
||||
mode, filename, body = articles.uudecode_group(i, file, fileout)
|
||||
|
||||
if file
|
||||
file.close
|
||||
fileout.close
|
||||
body = fullnameout
|
||||
end
|
||||
|
||||
if mode
|
||||
print "mode: #{mode}\n" if Debuglevel > 0
|
||||
print "filename: #{filename}\n"
|
||||
if options.has_key?("-L")
|
||||
print "longname\n" if Debuglevel > 1
|
||||
save_file(options["DATADIR"], i, body)
|
||||
else
|
||||
print "shortname\n" if Debuglevel > 1
|
||||
save_file(options["DATADIR"], filename, body)
|
||||
end
|
||||
end
|
||||
# rm file & fileout
|
||||
File.delete(fullname)
|
||||
else
|
||||
print "Not complete: #{i}\n"
|
||||
end
|
||||
print "\n"
|
||||
end
|
||||
end
|
||||
|
||||
articles.quit
|
||||
Loading…
Add table
Add a link
Reference in a new issue