diff --git a/trunk/ripnews/ripnews.rb b/trunk/ripnews/ripnews.rb index db28d0f..5d44fcf 100755 --- a/trunk/ripnews/ripnews.rb +++ b/trunk/ripnews/ripnews.rb @@ -560,7 +560,7 @@ def output_data(subject, mode, filename="", body="") if save_file("#{@config[group]["DATADIR"]}/#{group}", outfile, body) @newsrc_lock.synchronize { @articles.group_update_newsrc(subject) - @articles.save_newsrc unless @config[group].has_key?("-T") and @config[group]["-T"] + _save_newsrc(group) } end end @@ -673,8 +673,6 @@ def main @decode_threads = [] @newsrc_lock = Mutex.new profile_mem("#{group} start") -# puts "object count:" -# puts ObjectSpace.each_object(){} print "\nGetting articles for #{group}\n" @articles = Article.new(@config[group]["NNTPSERVER"], group, @config[group]["NEWSRCNAME"]) fill_preselector(group) @@ -683,73 +681,26 @@ def main profile_mem("#{group} articles read") - unless FileTest.directory?("#{@config[group]["DATADIR"]}/#{group}") or - Dir.mkdir("#{@config[group]["DATADIR"]}/#{group}", @config[group]["PERMISSION"].oct) - print "eeeps, couldn't create dir\n" - exit - end - for i in @articles.get_group_subjects.sort{|a, b| ward_sort(a, b)} - print "#{i}\n" if Debuglevel > 2 + _create_group_dir(group) + + for subj in @articles.get_group_subjects.sort{|a, b| ward_sort(a, b)} + print "#{subj}\n" if Debuglevel > 2 # explicitly mark as read - if @config[group].has_key?("-MR") and i =~ /#{@config[group]["-MR"]}/ - print "Marking '#{i}' as read\n" - @articles.group_update_newsrc(i) + if @config[group].has_key?("-MR") and subj =~ /#{@config[group]["-MR"]}/ + print "Marking '#{subj}' as read\n" + _mark_read(subj) # get the juicy bits - elsif !(@config[group].has_key?("-X") and i =~ /#{@config[group]["-X"]}/) and - i =~ /#{@config[group]["-I"]}/ - print "Match: #{i}\n" if Debuglevel > 0 - if @articles.group_is_complete(i) - skip = 0 - if @config[group].has_key?("PRIMARYTHRES") - if ( @articles.group_percentage_primary(i) < @config[group]["PRIMARYTHRES"].to_i ) - print "Only #{@articles.group_percentage_primary(i)}% of post on primary server, skipping #{i}\n" - skip = 1 - end - end - if @config[group].has_key?("FALLBACKTHRES") - if ( @articles.group_percentage_fallback(i) > @config[group]["FALLBACKTHRES"].to_i ) - print "#{@articles.group_percentage_fallback(i)}% of post only on fallback server, skipping #{i}\n" - skip = 1 - end - end - if ! skip - begin - if @articles.group_is_singlepart(i) - get_single(i, group) - elsif @articles.group_is_multipart(i) - get_multi(i, group) - end - rescue TempError, PermError, YencError - print "#{$!}\n" - print " Skipping article...\n" - #print "Caught #{$!.class}\n" - #print "Error: #{$!}\n" - end - end - else - print "Not complete: #{i}\n" - end - # if Mark Remaining Read is set do so - elsif @config[group].has_key?("-MRR") and @config[group]["-MRR"] and - !(@config[group].has_key?("-X") and i =~ /#{@config[group]["-X"]}/) and - !(i =~ /#{@config[group]["-I"]}/) - if i =~ /#{@config[group]["-I"]}/ - puts "fucking up here" - end - #print "Marking remaining '#{i}' as read\n" - @articles.group_update_newsrc(i) + elsif !(@config[group].has_key?("-X") and subj =~ /#{@config[group]["-X"]}/) and + subj =~ /#{@config[group]["-I"]}/ + print "Match: #{subj}\n" if Debuglevel > 0 + _get_article(subj, group) + else + _mark_remaining(subj, group) end end - # wait for threads if there are any - if ! @decode_threads.empty? - @articles.disconnect - puts "Waiting for decode threads..." - ThreadsWait.all_waits(@decode_threads){ |t| - puts "Thread #{t} has terminated" - } - puts "Decode threads all done" - end + _wait_for_threads + _save_newsrc(group) @articles.quit @articles = nil @@ -771,6 +722,91 @@ def ending unlock end +def _create_group_dir(group) + unless FileTest.directory?("#{@config[group]["DATADIR"]}/#{group}") or + Dir.mkdir("#{@config[group]["DATADIR"]}/#{group}", @config[group]["PERMISSION"].oct) + print "eeeps, couldn't create dir\n" + exit + end +end + +def _primary_thres_skip(subj, group) + if @config[group].has_key?("PRIMARYTHRES") + if ( @articles.group_percentage_primary(subj) < @config[group]["PRIMARYTHRES"].to_i ) + print "Only #{@articles.group_percentage_primary(subj)}% of post on primary server, skipping #{subj}\n" + return true + end + end + return false +end + +def _fallback_thres_skip(subj, group) + if @config[group].has_key?("FALLBACKTHRES") + if ( @articles.group_percentage_fallback(subj) > @config[group]["FALLBACKTHRES"].to_i ) + print "#{@articles.group_percentage_fallback(subj)}% of post only on fallback server, skipping #{subj}\n" + return true + end + end + return false +end + +def _mark_read(subj) + @articles.group_update_newsrc(subj) +end + +def _get_article(subj, group) + if @articles.group_is_complete(subj) + skip = false + skip = _primary_thres_skip(subj, group) ? true : skip + skip = _fallback_thres_skip(subj, group) ? true : skip + if ! skip + begin + if @articles.group_is_singlepart(subj) + get_single(subj, group) + elsif @articles.group_is_multipart(subj) + get_multi(subj, group) + end + rescue TempError, PermError, YencError + print "#{$!}\n" + print " Skipping article...\n" + #print "Caught #{$!.class}\n" + #print "Error: #{$!}\n" + end + end + else + print "Not complete: #{subj}\n" + end +end + +def _mark_remaining(subj, group) + # if Mark Remaining Read is set do so + if @config[group].has_key?("-MRR") and @config[group]["-MRR"] and + !(@config[group].has_key?("-X") and subj =~ /#{@config[group]["-X"]}/) and + !(subj =~ /#{@config[group]["-I"]}/) + if subj =~ /#{@config[group]["-I"]}/ + puts "fucking up here" + end + print "Marking remaining '#{subj}' as read\n" + @articles.group_update_newsrc(subj) + end +end + +def _wait_for_threads + # wait for threads if there are any + if ! @decode_threads.empty? + @articles.disconnect + puts "Waiting for decode threads..." + ThreadsWait.all_waits(@decode_threads){ |t| + puts "Thread #{t} has terminated" + } + puts "Decode threads all done" + end +end + +def _save_newsrc(group) + @articles.save_newsrc unless @config[group].has_key?("-T") and @config[group]["-T"] +end + startup main ending