ward/ripnews
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

- xover weer weggehaald, de oude manier was sneller

Browse source 
- code om niet alle gaten te proberen te vulln weer aangestoken
This commit is contained in:
Ward Wouts 2005-02-07 08:53:43 +00:00
parent e73d35d463
commit 78a364aae4
1 changed files with 33 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

65
trunk/ripnews/news/article.rb
View file

@ -1,4 +1,4 @@
# $Dwarf: article.rb,v 1.107 2005/02/05 12:35:50 ward Exp $ # $Dwarf: article.rb,v 1.108 2005/02/06 13:42:03 ward Exp $
# $Source$ # $Source$
# #
@ -180,20 +180,20 @@ def get_articles(cachedir=false)
# spul dat echt te oud is gaat nooit gevuld worden, dus doe ook geen poging het op te halen # spul dat echt te oud is gaat nooit gevuld worden, dus doe ook geen poging het op te halen
# wil wel wat ophalen aangezien logging aantoont dat er wel oudere articles gedownload worden # wil wel wat ophalen aangezien logging aantoont dat er wel oudere articles gedownload worden
# for server in @connections.keys for server in @connections.keys
# if @connections[server]["skip_ids"].max && @connections[server]["skip_ids"].max if @connections[server]["skip_ids"].max && @connections[server]["skip_ids"].max
# articles = @connections[server]["last"] - @connections[server]["first"] articles = @connections[server]["last"] - @connections[server]["first"]
# if articles > 10000 if articles > 10000
# fillerend = (@connections[server]["skip_ids"].max - (articles/10)).to_i fillerend = (@connections[server]["skip_ids"].max - (articles/10)).to_i
# else else
# fillerend = @connections[server]["skip_ids"].max - 1000 fillerend = @connections[server]["skip_ids"].max - 1000
# end end
# if @connections[server]["skip_ids"].min && fillerend > @connections[server]["skip_ids"].min if @connections[server]["skip_ids"].min && fillerend > @connections[server]["skip_ids"].min
# @connections[server]["skip_ids"] = @connections[server]["skip_ids"].union("#{@connections[server]["skip_ids"].min}-#{fillerend}") @connections[server]["skip_ids"] = @connections[server]["skip_ids"].union("#{@connections[server]["skip_ids"].min}-#{fillerend}")
# # p "filling #{@connections[server]["skip_ids"].min}-#{fillerend}" # p "filling #{@connections[server]["skip_ids"].min}-#{fillerend}"
# end end
# end end
# end end
for server in @connections.keys for server in @connections.keys
print " reading articles from server: #{server}\n" print " reading articles from server: #{server}\n"
@ -210,32 +210,32 @@ def get_articles(cachedir=false)
for i in rangelist.split(',') for i in rangelist.split(',')
print "i: #{i}\n" if Debuglevel > 2 print "i: #{i}\n" if Debuglevel > 2
begin begin
resp, xover_lines = get_xover(server, i) # resp, xover_lines = get_xover(server, i)
# resp, subj_lines = get_xhdr(server, i, "subject") resp, subj_lines = get_xhdr(server, i, "subject")
# resp, messid_lines = get_xhdr(server, i, "message-id") resp, messid_lines = get_xhdr(server, i, "message-id")
rescue TempError rescue TempError
printerr(server) printerr(server)
next next
end end
art = {} art = {}
xover_lines.collect{|x| # xover_lines.collect{|x|
art[x[0]] = {} unless art.has_key?(x[0])
art[x[0]]["subject"] = x[1]
art[x[0]]["messid"] = x[4]
print "art id: #{x[0]} subj: #{x[1]}\n" if Debuglevel > 2
print "art id: #{x[0]} messid: #{x[4]}\n" if Debuglevel > 2
}
# subj_lines.collect{|x|
# art[x[0]] = {} unless art.has_key?(x[0]) # art[x[0]] = {} unless art.has_key?(x[0])
# art[x[0]]["subject"] = x[1] # art[x[0]]["subject"] = x[1]
# art[x[0]]["messid"] = x[4]
# print "art id: #{x[0]} subj: #{x[1]}\n" if Debuglevel > 2 # print "art id: #{x[0]} subj: #{x[1]}\n" if Debuglevel > 2
# print "art id: #{x[0]} messid: #{x[4]}\n" if Debuglevel > 2
# } # }
# messid_lines.collect{|x| subj_lines.collect{|x|
# art[x[0]] = {} unless art.has_key?(x[0]) art[x[0]] = {} unless art.has_key?(x[0])
# art[x[0]]["messid"] = x[1] art[x[0]]["subject"] = x[1]
# print "art id: #{x[0]} messid: #{x[1]}\n" if Debuglevel > 2 print "art id: #{x[0]} subj: #{x[1]}\n" if Debuglevel > 2
# } }
messid_lines.collect{|x|
art[x[0]] = {} unless art.has_key?(x[0])
art[x[0]]["messid"] = x[1]
print "art id: #{x[0]} messid: #{x[1]}\n" if Debuglevel > 2
}
for id in art.keys for id in art.keys
if art[id].has_key?("subject") and art[id].has_key?("messid") if art[id].has_key?("subject") and art[id].has_key?("messid")
print "adding: #{art[id]["messid"]}, #{id}, #{server}, #{art[id]["subject"]}\n" if Debuglevel > 2 print "adding: #{art[id]["messid"]}, #{id}, #{server}, #{art[id]["subject"]}\n" if Debuglevel > 2
@ -249,7 +249,8 @@ def get_articles(cachedir=false)
cache_add(cachedir, id, art[id]["messid"], art[id]["subject"], server) cache_add(cachedir, id, art[id]["messid"], art[id]["subject"], server)
end end
end end
headerlines += xover_lines.length # headerlines += xover_lines.length
headerlines += subj_lines.length
if headerlines >= 500 if headerlines >= 500
cache_save(cachedir, server) cache_save(cachedir, server)
headerlines = 0 headerlines = 0