commit before mofe to pure git

trunk/ripnews/encode/tests/uu_test.rb

@@ -73,8 +73,28 @@ def test4
 	end
 end
 
+def test5
+	print "Test 5: decoding a downloaded file\n"
+	file = File.open("testdata-download.uu", "r")
+	tmpfile = Tempfile.new("uutmp")
+	tmpfile.sync=true
+	mode, filename, body = UUEncode.uudecode(file, tmpfile)
+	if mode != "600"
+		print "	Failed, mode should be 600, but is #{mode}\n"
+	elsif filename != "testdata-dowload.uu"
+		print "	Failed, filename should be \"testdata\", but is \"#{filename}\"\n"
+	elsif ! FileUtils.compare_file("testdata-download-ydecoded.rar", tmpfile.path)
+		print "	Failed, result doesn't match reference data\n"
+	else
+		print "	Succesful\n"
+	end
+	file.close
+	tmpfile.close
+end
+
 test1
 test2
 test3
 test4
+test5
 

trunk/ripnews/encode/y_enc.rb

@@ -0,0 +1,151 @@
+require 'zlib'
+
+# yEnc
+#
+# This gem allows you to decode and encode files using the yenc standard.
+
+class YEnc
+
+  attr_reader :filepath, :outputpath, :filename, :filesize, :line
+
+  def initialize filepath, outputpath
+    @filepath = filepath
+    @outputpath = outputpath
+  end
+
+  def crc32
+    @crc32.upcase.strip
+  end
+
+  # Encode file into a yenc text file
+  def encode_to_file outputfilename
+    outputfile = File.new(@outputpath + outputfilename, "w")
+    outputfile.puts "=ybegin size=#{File.size?(@filepath)} line=128 name=#{File.basename @filepath}\n"
+    special = { 0 => nil, 10 => nil, 13 => nil, 61 => nil }
+    File.open(@filepath,'rb') do |f|
+      until f.eof?
+        #Read in 128 bytes at a time
+        buffer = f.read(128)
+        buffer.each_byte do |byte|
+          char_to_write = (byte + 42) % 256
+          if special.has_key?(char_to_write)
+            outputfile.putc '='
+            char_to_write = (char_to_write + 64) % 256
+          end
+          outputfile.putc char_to_write
+        end
+        outputfile.puts "\n"
+      end
+    end
+    outputfile.puts "=yend size=312860 crc32=#{file_crc32(@filepath).upcase}\n"
+    outputfile.close
+  end
+
+  # method only encodes given file and returns yenc encoded string; nothing more, nothing less
+  # Author: Tadeus Dobrovolskij
+  def encode
+    sio = StringIO.new("","w:ASCII-8BIT")
+    special = { 0 => nil, 10 => nil, 13 => nil, 61 => nil }
+    File.open(@filepath,'rb') do |b|
+      until b.eof?
+        buffer = b.read(128)
+        buffer.each_byte do |byte|
+          char_to_write = (byte + 42) % 256
+          if special.has_key?(char_to_write)
+            sio.putc '='
+            char_to_write = (char_to_write + 64) % 256
+          end
+          sio.putc char_to_write
+        end
+        sio.puts "\n"
+      end
+    end
+    result = sio.string
+    sio.close
+    return result
+  end
+
+  def decode
+    if is_yenc?
+            #Continue decoding
+      begin_read = false
+
+      File.open(@filepath, 'r').each_line do |line|
+
+        if begin_read
+          if line.start_with?("=yend")
+            breakdown_endline line
+            begin_read=false
+            break #stop looking through the file we are done
+          end
+          #end of reading lines
+
+          #puts "LINE COUNT: #{line.length}"
+          #Decode and write to binary file
+          esc = false
+          line.each_byte do |c|
+            next if c == 13 or c == 10
+
+            if c == 61 and not esc #escape character hit goto the next one
+              esc = true
+              next
+            else
+              if esc
+                esc = false
+                c = c - 64
+              end
+
+              if c.between?(0,41)
+                decoded = c + 214
+              else
+                decoded = c - 42
+              end
+            end
+            @new_file.putc decoded
+          end
+
+        else
+          if line.start_with?("=ybegin") #This is the begin size
+            breakdown_header line
+            begin_read = true
+            next
+          end
+        end
+      end
+    else
+      false
+    end
+    @new_file.close
+  end
+
+  #Does this pass the crc32 check
+  def pass_crc32?
+    crc32 = file_crc32 @outputpath + @filename
+    crc32.eql?(@crc32.downcase.strip)
+  end
+
+  # Get the CRC32 for a file
+  def file_crc32 filepath
+    f = nil
+    File.open( filepath, "rb") { |h| f = h.read }
+    Zlib.crc32(f,0).to_s(16)
+  end
+
+  private
+
+  def is_yenc?
+    File.read(@filepath).include?("=ybegin")
+  end
+
+  def breakdown_endline line
+    @crc32 = line[/crc32=(.*)/,1] if @crc32.nil?
+  end
+
+  def breakdown_header line
+    @filename=line[/name=(.*)/,1] if @filename.nil?
+    @filesize =line[/size=([^\s]+)/,1] if @filesize.nil?
+    @line=line[/line=([^\s]+)/,1] if @line.nil?
+    @new_file = File.new(@outputpath + @filename, "wb")
+  end
+
+end

trunk/ripnews/encode/yenc.rb

@@ -26,7 +26,7 @@ Encoding.default_internal="ISO-8859-1"
 #Encoding.default_external="ASCII-8BIT"
 #Encoding.default_internal="ASCII-8BIT"
 
-Debuglevel = 0
+Debuglevel = 1
 @@ymap = {}
 
 def ydecode(data, outfile=nil)
@@ -93,10 +93,10 @@ def _ydecode_file(file, outfile)
 
 	while (! file.eof)
 		line = file.gets
-		print "line: #{line}" if Debuglevel > 0
+		print "line: #{line}" if Debuglevel > 1
 		if  line.match(/^\=ybegin\s+(.*line\=.*)/)
 			m = $1
-			puts "ybegin match; rest: #{m}"  if Debuglevel > 0
+			puts "	#{Thread.current.inspect} ybegin match; rest: #{m}"  if Debuglevel > 0
 			if m.match(/^\s*(part\=(\d+)\s+)?(total\=(\d+)\s+)?(line\=(\d+))(\s*size\=(\d+))(\s*name=(.*?\S))\s*$/)
 				part = $2.to_i
 				total = $4.to_i
@@ -154,6 +154,7 @@ def _ydecode_file(file, outfile)
 			end
 			search_begin = true
 			bytes = 0
+			puts "	#{Thread.current.inspect} yended"  if Debuglevel > 0
 			next
 		end
 		if search_begin && line.match(/^\=ybegin\s+(.*)\Z/)
@@ -200,9 +201,9 @@ def _ydecode_file(file, outfile)
 #		end
 
 		if !skip
-			puts "line: #{line}" if Debuglevel > 0
+			puts "line: #{line}" if Debuglevel > 1
 			ostr = _ydecode_line(line)
-			puts "ostr: #{ostr}" if Debuglevel > 0
+			puts "ostr: #{ostr}" if Debuglevel > 1
 			outfile << ostr
 			bytes += ostr.length
 		end
@@ -228,14 +229,14 @@ def _ydecode_array(data)
 	while (i < data.length)
 		if  data[i].match(/^\=ybegin\s+(.*line\=.*)/)
 			m = $1
-			puts "ybegin match; rest: #{m}"  if Debuglevel > 0
+			puts "	#{Thread.current.inspect} ybegin match; rest: #{m}"  if Debuglevel > 0
 			if m.match(/^\s*(part\=(\d+)\s+)?(total\=(\d+)\s+)?(line\=(\d+))(\s*size\=(\d+))(\s*name=(.*?\S))\s*$/)
 				part = $2.to_i
 				total = $4.to_i
 				linesize = $6.to_i
 				size = $8.to_i
 				filename = $10
-				puts "found beginning, linesize = #{linesize}, size = #{size}, filename = #{filename}" if Debuglevel > 0
+				puts "	#{Thread.current.inspect} found beginning, linesize = #{linesize}, size = #{size}, filename = #{filename}" if Debuglevel > 0
 				i += 1
 				break
 			else
@@ -255,13 +256,13 @@ def _ydecode_array(data)
 		line.chomp!("\n")
 		line.chomp!("\r")
 		puts "at #{i} need to go to #{data.length}" if Debuglevel > 1
-		print "line: #{line}" if Debuglevel > 0
+		print "line: #{line}" if Debuglevel > 1
 		i += 1
 		if line.match(/^\=yend(\s+size=(\d+))(\s+crc32=(\S+))?/)
 			size = $2.to_i
 			crc = $4
 			if size != decode.length
-				puts "#{Thread.current.inspect} size mismatch, was #{decode.length}, should be #{size}"
+				puts "	#{Thread.current.inspect} size mismatch, was #{decode.length}, should be #{size}"
 			end
 			dec = [ decode ]
 			return mode, filename, dec
@@ -292,7 +293,7 @@ def _ydecode_array(data)
 #		end
 
 		if !skip
-			print "line: #{line}" if Debuglevel > 0
+			print "line: #{line}" if Debuglevel > 1
 			ostr = _ydecode_line(line)
 			decode << ostr
 			bytes += ostr.length

trunk/ripnews/news/article.rb

@@ -85,7 +85,7 @@ def quit
 	@thr.exit
 	begin
 	super
-	rescue EOFError, Errno::EPIPE
+	rescue EOFError, Errno::EPIPE, IOError
 	end
 end
 

trunk/ripnews/ripnews.rb

@@ -78,7 +78,7 @@ def aprofile_mem(group)
 end
 ###########################################################################
 
-Debuglevel = 0
+Debuglevel = 1
 @tstart = Time.now
 
 def save_file(dir, name, data)
@@ -110,9 +110,9 @@ def save_file(dir, name, data)
 				end
 			when "Array"
 				puts "	Saving to #{dir}/#{newname}"
-				if file = File.new("#{dir}/#{newname}", "w", 0644)
+				if file = File.new("#{dir}/#{newname}", "w:ascii-8bit", 0644)
 					puts "	Saved as: '#{newname}'"
-					data.collect{|i| file.print "#{i}"}
+					data.collect{|i| file.write i }
 				else
 					puts "couldn't open file for writeing"
 					return false
@@ -563,6 +563,7 @@ def get_multi(subj, group)
 			end
 
 			if tfile
+				@decode_file_lock.synchronize {
 					# horrible cheat to not lose the outputted file
 					tbody = tfileout.path
 					tbodybase = tbody.sub(/\/[^\/]*$/, "/ripnewsdecode")
@@ -574,6 +575,7 @@ def get_multi(subj, group)
 					tbody = "#{tbodybase}-#{i}"
 					tfile.close
 					tfileout.close(false)
+				}
 			end
 			output_data(tsubj, tmode, tfilename, tbody)
 		end # thread end
@@ -753,6 +755,7 @@ def main
 		end
 		@decode_threads = []
 		@newsrc_lock = Mutex.new
+		@decode_file_lock = Mutex.new
 		profile_mem("#{group} start")
 		puts "\nGetting articles for #{group}"
 		@articles = Article.new(@config[group]["NNTPSERVER"], group, @config[group]["NEWSRCNAME"], @config[group]["MAXAGE"])