ward/ripnews
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add threading

Browse source
This commit is contained in:
Ward Wouts 2005-01-30 14:24:47 +00:00
parent bb4ae13cfa
commit d68c916e89
1 changed files with 84 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

120
trunk/ripnews/ripnews.rb
View file

@ -1,10 +1,10 @@
#!/usr/local/bin/ruby -w
# $Dwarf: ripnews.rb,v 1.87 2005/01/28 20:13:54 ward Exp $
# $Dwarf: ripnews.rb,v 1.88 2005/01/29 18:42:30 ward Exp $
# $Source$
#
# Copyright (c) 2002, 2003, 2004 Ward Wouts <ward@wouts.nl>
# Copyright (c) 2002, 2003, 2004, 2005 Ward Wouts <ward@wouts.nl>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
@ -355,12 +355,18 @@ def get_single(subj, group)
if UUEncode.is_uuencoded(body)
filename = UUEncode.get_filename(body)
print " filename #{filename}\n"
return false unless check_ext(group, filename, "s", subj)
unless check_ext(group, filename, "s", subj)
print " Skipping article...\n"
return false
end
print " UUDecoding...\n"
mode, filename, body = UUEncode.uudecode(body)
elsif YEnc.is_yencoded(body)
filename = YEnc.get_filename(body)
return false unless check_ext(group, filename, "s", subj)
unless check_ext(group, filename, "s", subj)
print " Skipping article...\n"
return false
end
print " YDecoding...\n"
mode, filename, body = YEnc.ydecode(body)
else
@ -368,6 +374,7 @@ def get_single(subj, group)
return false
end
if mode == false
print " Decoding failed skipping article...\n"
return false
end
output_data(subj, mode, filename, body)
@ -381,14 +388,20 @@ def get_multi(subj, group)
if UUEncode.is_uuencoded(body)
filename = UUEncode.get_filename(body)
print " filename #{filename}\n"
return false unless check_ext(group, filename, "m", subj)
unless check_ext(group, filename, "m", subj)
print " Skipping article...\n"
return false
end
print " UUDecoding...\n"
mode, filename, body = UUEncode.uudecode(body)
elsif YEnc.is_yencoded(body)
print "yencc\n"
filename = YEnc.get_filename(body)
print "filename #{filename}\n"
return false unless check_ext(group, filename, "m", subj)
unless check_ext(group, filename, "m", subj)
print " Skipping article...\n"
return false
end
print " YDecoding...\n"
begin
mode, filename, body = YEnc.ydecode(body)
@ -403,6 +416,7 @@ def get_multi(subj, group)
return false
end
if mode == false
print " Decoding failed skipping article...\n"
return false
end
output_data(subj, mode, filename, body)
@ -414,46 +428,72 @@ def get_multi(subj, group)
filename = UUEncode.get_filename(body)
print " filename #{filename}\n"
return false unless check_ext(group, filename, "m", subj)
unless check_ext(group, filename, "m", subj)
print " Skipping article...\n"
return false
end
elsif YEnc.is_yencoded(body)
print "yencc\n"
filename = YEnc.get_filename(body)
print "filename #{filename}\n"
return false unless check_ext(group, filename, "m", subj)
unless check_ext(group, filename, "m", subj)
print " Skipping article...\n"
return false
end
end
file = Tempfile.new("riptmp", @config[group]["TEMPDIR"])
body.collect{|x| file.print "#{x}\n"}
return false unless @articles.get_group_body_rest(subj, file)
unless @articles.get_group_body_rest(subj, file)
print " Skipping article...\n"
return false
end
fileout = Tempfile.new("riptmp", @config[group]["TEMPDIR"])
# I think a thread should start about here
@decode_threads << Thread.new(body, file, fileout, subj) do |tbody, tfile,tfileout, tsubj|
# imediately stop to continue with main program
Thread.stop
puts "inside thread\n"
if UUEncode.is_uuencoded(tbody)
print " UUDecoding...\n"
mode, filename, body = UUEncode.uudecode(tfile, tfileout)
elsif YEnc.is_yencoded(body)
print " YDecoding...\n"
mode, filename, body = YEnc.ydecode(tfile, tfileout)
end
if mode == false
print " Decoding failed skipping article...\n"
return false
end
if UUEncode.is_uuencoded(body)
print " UUDecoding...\n"
mode, filename, body = UUEncode.uudecode(file, fileout)
elsif YEnc.is_yencoded(body)
print " YDecoding...\n"
mode, filename, body = YEnc.ydecode(file, fileout)
# horrible cheat to not lose the outputted file
tbody = tfileout.path
bodybase = tbody.sub(/\/[^\/]*$/, "/ripnewsdecode")
i = 1
while FileTest.exists?("#{bodybase}-#{i}")
i += 1
end
File.move(tbody, "#{bodybase}-#{i}")
tbody = "#{bodybase}-#{i}"
file.close
fileout.close(false)
output_data(tsubj, mode, filename, tbody)
end
if mode == false
return false
end
# horrible cheat to not lose the outputted file
body = fileout.path
bodybase = body.sub(/\/[^\/]*$/, "/ripnewsdecode")
i = 1
while FileTest.exists?("#{bodybase}-#{i}")
i += 1
end
File.move(body, "#{bodybase}-#{i}")
body = "#{bodybase}-#{i}"
file.close
fileout.close(false)
output_data(subj, mode, filename, body)
# thread could end here
@decode_threads.each{ |thr|
if thr.status == "sleep" # and fire up the threads again
thr.run
elsif thr.status == "false" # remove finished threads
thr.join
else
p thr.status
end
}
puts "ouside thread\n"
return true
else
print " Unknown encoding (not UU, not yEnc), skipping...\n"
@ -605,6 +645,7 @@ end
def main
profile_mem("out side of loop still")
for group in @config.keys.sort
@decode_threads = []
profile_mem("#{group} start")
puts "object count:"
puts ObjectSpace.each_object(){}
@ -634,12 +675,9 @@ def main
if @articles.group_is_complete(i)
begin
if @articles.group_is_singlepart(i)
succes = get_single(i, group)
get_single(i, group)
elsif @articles.group_is_multipart(i)
succes = get_multi(i, group)
end
if succes == false
print " Skipping article...\n"
get_multi(i, group)
end
#rescue Article::TempError, Article::PermError
rescue TempError, PermError
@ -654,6 +692,14 @@ def main
end
end
end
# hier wachten op evt. threads...
if @decode_threads
puts "Waiting for decode threads..."
@decode_threads.each{|thr| thr.join}
puts "Decode threads all done"
end
@articles.quit
@articles = nil
profile_mem("#{group} pre-GC")