diff --git a/trunk/ripnews/ripnews.rb b/trunk/ripnews/ripnews.rb index 86c9fc4..d1033df 100755 --- a/trunk/ripnews/ripnews.rb +++ b/trunk/ripnews/ripnews.rb @@ -1,10 +1,10 @@ #!/usr/local/bin/ruby -w -# $Dwarf: ripnews.rb,v 1.87 2005/01/28 20:13:54 ward Exp $ +# $Dwarf: ripnews.rb,v 1.88 2005/01/29 18:42:30 ward Exp $ # $Source$ # -# Copyright (c) 2002, 2003, 2004 Ward Wouts +# Copyright (c) 2002, 2003, 2004, 2005 Ward Wouts # # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above @@ -355,12 +355,18 @@ def get_single(subj, group) if UUEncode.is_uuencoded(body) filename = UUEncode.get_filename(body) print " filename #{filename}\n" - return false unless check_ext(group, filename, "s", subj) + unless check_ext(group, filename, "s", subj) + print " Skipping article...\n" + return false + end print " UUDecoding...\n" mode, filename, body = UUEncode.uudecode(body) elsif YEnc.is_yencoded(body) filename = YEnc.get_filename(body) - return false unless check_ext(group, filename, "s", subj) + unless check_ext(group, filename, "s", subj) + print " Skipping article...\n" + return false + end print " YDecoding...\n" mode, filename, body = YEnc.ydecode(body) else @@ -368,6 +374,7 @@ def get_single(subj, group) return false end if mode == false + print " Decoding failed skipping article...\n" return false end output_data(subj, mode, filename, body) @@ -381,14 +388,20 @@ def get_multi(subj, group) if UUEncode.is_uuencoded(body) filename = UUEncode.get_filename(body) print " filename #{filename}\n" - return false unless check_ext(group, filename, "m", subj) + unless check_ext(group, filename, "m", subj) + print " Skipping article...\n" + return false + end print " UUDecoding...\n" mode, filename, body = UUEncode.uudecode(body) elsif YEnc.is_yencoded(body) print "yencc\n" filename = YEnc.get_filename(body) print "filename #{filename}\n" - return false unless check_ext(group, filename, "m", subj) + unless check_ext(group, filename, "m", subj) + print " Skipping article...\n" + return false + end print " YDecoding...\n" begin mode, filename, body = YEnc.ydecode(body) @@ -403,6 +416,7 @@ def get_multi(subj, group) return false end if mode == false + print " Decoding failed skipping article...\n" return false end output_data(subj, mode, filename, body) @@ -414,46 +428,72 @@ def get_multi(subj, group) filename = UUEncode.get_filename(body) print " filename #{filename}\n" return false unless check_ext(group, filename, "m", subj) + unless check_ext(group, filename, "m", subj) + print " Skipping article...\n" + return false + end elsif YEnc.is_yencoded(body) print "yencc\n" filename = YEnc.get_filename(body) print "filename #{filename}\n" - return false unless check_ext(group, filename, "m", subj) + unless check_ext(group, filename, "m", subj) + print " Skipping article...\n" + return false + end end file = Tempfile.new("riptmp", @config[group]["TEMPDIR"]) body.collect{|x| file.print "#{x}\n"} - return false unless @articles.get_group_body_rest(subj, file) + + unless @articles.get_group_body_rest(subj, file) + print " Skipping article...\n" + return false + end fileout = Tempfile.new("riptmp", @config[group]["TEMPDIR"]) # I think a thread should start about here - - if UUEncode.is_uuencoded(body) - print " UUDecoding...\n" - mode, filename, body = UUEncode.uudecode(file, fileout) - elsif YEnc.is_yencoded(body) - print " YDecoding...\n" - mode, filename, body = YEnc.ydecode(file, fileout) + @decode_threads << Thread.new(body, file, fileout, subj) do |tbody, tfile,tfileout, tsubj| + # imediately stop to continue with main program + Thread.stop + puts "inside thread\n" + if UUEncode.is_uuencoded(tbody) + print " UUDecoding...\n" + mode, filename, body = UUEncode.uudecode(tfile, tfileout) + elsif YEnc.is_yencoded(body) + print " YDecoding...\n" + mode, filename, body = YEnc.ydecode(tfile, tfileout) + end + if mode == false + print " Decoding failed skipping article...\n" + return false + end + + # horrible cheat to not lose the outputted file + tbody = tfileout.path + bodybase = tbody.sub(/\/[^\/]*$/, "/ripnewsdecode") + i = 1 + while FileTest.exists?("#{bodybase}-#{i}") + i += 1 + end + File.move(tbody, "#{bodybase}-#{i}") + tbody = "#{bodybase}-#{i}" + file.close + fileout.close(false) + + output_data(tsubj, mode, filename, tbody) end - if mode == false - return false - end - - # horrible cheat to not lose the outputted file - body = fileout.path - bodybase = body.sub(/\/[^\/]*$/, "/ripnewsdecode") - i = 1 - while FileTest.exists?("#{bodybase}-#{i}") - i += 1 - end - File.move(body, "#{bodybase}-#{i}") - body = "#{bodybase}-#{i}" - file.close - fileout.close(false) - - output_data(subj, mode, filename, body) - # thread could end here + @decode_threads.each{ |thr| + if thr.status == "sleep" # and fire up the threads again + thr.run + elsif thr.status == "false" # remove finished threads + thr.join + else + p thr.status + end + } + puts "ouside thread\n" + return true else print " Unknown encoding (not UU, not yEnc), skipping...\n" @@ -605,6 +645,7 @@ end def main profile_mem("out side of loop still") for group in @config.keys.sort + @decode_threads = [] profile_mem("#{group} start") puts "object count:" puts ObjectSpace.each_object(){} @@ -634,12 +675,9 @@ def main if @articles.group_is_complete(i) begin if @articles.group_is_singlepart(i) - succes = get_single(i, group) + get_single(i, group) elsif @articles.group_is_multipart(i) - succes = get_multi(i, group) - end - if succes == false - print " Skipping article...\n" + get_multi(i, group) end #rescue Article::TempError, Article::PermError rescue TempError, PermError @@ -654,6 +692,14 @@ def main end end end + + # hier wachten op evt. threads... + if @decode_threads + puts "Waiting for decode threads..." + @decode_threads.each{|thr| thr.join} + puts "Decode threads all done" + end + @articles.quit @articles = nil profile_mem("#{group} pre-GC")