readablity cleanups
This commit is contained in:
parent
4459486853
commit
c3f4168506
1 changed files with 101 additions and 65 deletions
|
|
@ -560,7 +560,7 @@ def output_data(subject, mode, filename="", body="")
|
|||
if save_file("#{@config[group]["DATADIR"]}/#{group}", outfile, body)
|
||||
@newsrc_lock.synchronize {
|
||||
@articles.group_update_newsrc(subject)
|
||||
@articles.save_newsrc unless @config[group].has_key?("-T") and @config[group]["-T"]
|
||||
_save_newsrc(group)
|
||||
}
|
||||
end
|
||||
end
|
||||
|
|
@ -673,8 +673,6 @@ def main
|
|||
@decode_threads = []
|
||||
@newsrc_lock = Mutex.new
|
||||
profile_mem("#{group} start")
|
||||
# puts "object count:"
|
||||
# puts ObjectSpace.each_object(){}
|
||||
print "\nGetting articles for #{group}\n"
|
||||
@articles = Article.new(@config[group]["NNTPSERVER"], group, @config[group]["NEWSRCNAME"])
|
||||
fill_preselector(group)
|
||||
|
|
@ -683,73 +681,26 @@ def main
|
|||
|
||||
profile_mem("#{group} articles read")
|
||||
|
||||
unless FileTest.directory?("#{@config[group]["DATADIR"]}/#{group}") or
|
||||
Dir.mkdir("#{@config[group]["DATADIR"]}/#{group}", @config[group]["PERMISSION"].oct)
|
||||
print "eeeps, couldn't create dir\n"
|
||||
exit
|
||||
end
|
||||
for i in @articles.get_group_subjects.sort{|a, b| ward_sort(a, b)}
|
||||
print "#{i}\n" if Debuglevel > 2
|
||||
_create_group_dir(group)
|
||||
|
||||
for subj in @articles.get_group_subjects.sort{|a, b| ward_sort(a, b)}
|
||||
print "#{subj}\n" if Debuglevel > 2
|
||||
# explicitly mark as read
|
||||
if @config[group].has_key?("-MR") and i =~ /#{@config[group]["-MR"]}/
|
||||
print "Marking '#{i}' as read\n"
|
||||
@articles.group_update_newsrc(i)
|
||||
if @config[group].has_key?("-MR") and subj =~ /#{@config[group]["-MR"]}/
|
||||
print "Marking '#{subj}' as read\n"
|
||||
_mark_read(subj)
|
||||
# get the juicy bits
|
||||
elsif !(@config[group].has_key?("-X") and i =~ /#{@config[group]["-X"]}/) and
|
||||
i =~ /#{@config[group]["-I"]}/
|
||||
print "Match: #{i}\n" if Debuglevel > 0
|
||||
if @articles.group_is_complete(i)
|
||||
skip = 0
|
||||
if @config[group].has_key?("PRIMARYTHRES")
|
||||
if ( @articles.group_percentage_primary(i) < @config[group]["PRIMARYTHRES"].to_i )
|
||||
print "Only #{@articles.group_percentage_primary(i)}% of post on primary server, skipping #{i}\n"
|
||||
skip = 1
|
||||
end
|
||||
end
|
||||
if @config[group].has_key?("FALLBACKTHRES")
|
||||
if ( @articles.group_percentage_fallback(i) > @config[group]["FALLBACKTHRES"].to_i )
|
||||
print "#{@articles.group_percentage_fallback(i)}% of post only on fallback server, skipping #{i}\n"
|
||||
skip = 1
|
||||
end
|
||||
end
|
||||
if ! skip
|
||||
begin
|
||||
if @articles.group_is_singlepart(i)
|
||||
get_single(i, group)
|
||||
elsif @articles.group_is_multipart(i)
|
||||
get_multi(i, group)
|
||||
end
|
||||
rescue TempError, PermError, YencError
|
||||
print "#{$!}\n"
|
||||
print " Skipping article...\n"
|
||||
#print "Caught #{$!.class}\n"
|
||||
#print "Error: #{$!}\n"
|
||||
end
|
||||
end
|
||||
else
|
||||
print "Not complete: #{i}\n"
|
||||
end
|
||||
# if Mark Remaining Read is set do so
|
||||
elsif @config[group].has_key?("-MRR") and @config[group]["-MRR"] and
|
||||
!(@config[group].has_key?("-X") and i =~ /#{@config[group]["-X"]}/) and
|
||||
!(i =~ /#{@config[group]["-I"]}/)
|
||||
if i =~ /#{@config[group]["-I"]}/
|
||||
puts "fucking up here"
|
||||
end
|
||||
#print "Marking remaining '#{i}' as read\n"
|
||||
@articles.group_update_newsrc(i)
|
||||
elsif !(@config[group].has_key?("-X") and subj =~ /#{@config[group]["-X"]}/) and
|
||||
subj =~ /#{@config[group]["-I"]}/
|
||||
print "Match: #{subj}\n" if Debuglevel > 0
|
||||
_get_article(subj, group)
|
||||
else
|
||||
_mark_remaining(subj, group)
|
||||
end
|
||||
end
|
||||
|
||||
# wait for threads if there are any
|
||||
if ! @decode_threads.empty?
|
||||
@articles.disconnect
|
||||
puts "Waiting for decode threads..."
|
||||
ThreadsWait.all_waits(@decode_threads){ |t|
|
||||
puts "Thread #{t} has terminated"
|
||||
}
|
||||
puts "Decode threads all done"
|
||||
end
|
||||
_wait_for_threads
|
||||
_save_newsrc(group)
|
||||
|
||||
@articles.quit
|
||||
@articles = nil
|
||||
|
|
@ -771,6 +722,91 @@ def ending
|
|||
unlock
|
||||
end
|
||||
|
||||
def _create_group_dir(group)
|
||||
unless FileTest.directory?("#{@config[group]["DATADIR"]}/#{group}") or
|
||||
Dir.mkdir("#{@config[group]["DATADIR"]}/#{group}", @config[group]["PERMISSION"].oct)
|
||||
print "eeeps, couldn't create dir\n"
|
||||
exit
|
||||
end
|
||||
end
|
||||
|
||||
def _primary_thres_skip(subj, group)
|
||||
if @config[group].has_key?("PRIMARYTHRES")
|
||||
if ( @articles.group_percentage_primary(subj) < @config[group]["PRIMARYTHRES"].to_i )
|
||||
print "Only #{@articles.group_percentage_primary(subj)}% of post on primary server, skipping #{subj}\n"
|
||||
return true
|
||||
end
|
||||
end
|
||||
return false
|
||||
end
|
||||
|
||||
def _fallback_thres_skip(subj, group)
|
||||
if @config[group].has_key?("FALLBACKTHRES")
|
||||
if ( @articles.group_percentage_fallback(subj) > @config[group]["FALLBACKTHRES"].to_i )
|
||||
print "#{@articles.group_percentage_fallback(subj)}% of post only on fallback server, skipping #{subj}\n"
|
||||
return true
|
||||
end
|
||||
end
|
||||
return false
|
||||
end
|
||||
|
||||
def _mark_read(subj)
|
||||
@articles.group_update_newsrc(subj)
|
||||
end
|
||||
|
||||
def _get_article(subj, group)
|
||||
if @articles.group_is_complete(subj)
|
||||
skip = false
|
||||
skip = _primary_thres_skip(subj, group) ? true : skip
|
||||
skip = _fallback_thres_skip(subj, group) ? true : skip
|
||||
if ! skip
|
||||
begin
|
||||
if @articles.group_is_singlepart(subj)
|
||||
get_single(subj, group)
|
||||
elsif @articles.group_is_multipart(subj)
|
||||
get_multi(subj, group)
|
||||
end
|
||||
rescue TempError, PermError, YencError
|
||||
print "#{$!}\n"
|
||||
print " Skipping article...\n"
|
||||
#print "Caught #{$!.class}\n"
|
||||
#print "Error: #{$!}\n"
|
||||
end
|
||||
end
|
||||
else
|
||||
print "Not complete: #{subj}\n"
|
||||
end
|
||||
end
|
||||
|
||||
def _mark_remaining(subj, group)
|
||||
# if Mark Remaining Read is set do so
|
||||
if @config[group].has_key?("-MRR") and @config[group]["-MRR"] and
|
||||
!(@config[group].has_key?("-X") and subj =~ /#{@config[group]["-X"]}/) and
|
||||
!(subj =~ /#{@config[group]["-I"]}/)
|
||||
if subj =~ /#{@config[group]["-I"]}/
|
||||
puts "fucking up here"
|
||||
end
|
||||
print "Marking remaining '#{subj}' as read\n"
|
||||
@articles.group_update_newsrc(subj)
|
||||
end
|
||||
end
|
||||
|
||||
def _wait_for_threads
|
||||
# wait for threads if there are any
|
||||
if ! @decode_threads.empty?
|
||||
@articles.disconnect
|
||||
puts "Waiting for decode threads..."
|
||||
ThreadsWait.all_waits(@decode_threads){ |t|
|
||||
puts "Thread #{t} has terminated"
|
||||
}
|
||||
puts "Decode threads all done"
|
||||
end
|
||||
end
|
||||
|
||||
def _save_newsrc(group)
|
||||
@articles.save_newsrc unless @config[group].has_key?("-T") and @config[group]["-T"]
|
||||
end
|
||||
|
||||
startup
|
||||
main
|
||||
ending
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue