ripnews/trunk/ripnews/ripnews.rb

#!/usr/local/bin/ruby -w

# $Dwarf: ripnews.rb,v 1.37 2002/11/05 09:33:41 ward Exp $
# $Source$

require 'date'
require 'getoptlong'
require 'news/article'
require 'news/newsrc'
require 'tempfile'


###########################################################################

Debuglevel = 0

def save_file(dir, name, data)
	print "savename: #{name}\n" if Debuglevel > 1
	nname = name.gsub(/\//, "-")
	print "nname: #{nname}\n" if Debuglevel > 1
	nname.sub!(/\s*$/, "")
	nname.sub!(/^\s*/, "")
	nname.sub!(/^\.*/, "")
	newname = nname
	count = 1
	d = Date.today
	date = "#{d.year}#{d.month}#{d.mday}"
	while FileTest.exists?("#{dir}/#{newname}")
		newname = "#{nname}-<#{date}.#{count}>"
		count += 1
	end
	print "name: #{newname}\n" if Debuglevel > 1

	case data.type.to_s
		when "String"
			begin
				if File.rename(data, "#{dir}/#{newname}")
					print "	Saving as: '#{newname}'\n"
				else
					print "couldn't rename tempfile\n"
					return false
				end
			rescue Errno::ENOENT
				print "Caught Errno::ENOENT (save_file)\n"
				print "Error: #{$!}\n"
				print "What the #@$$ happended?\n"
				return false
			end
		when "Array"
			if file = File.new("#{dir}/#{newname}", "w", "0644")
				print "	Saving as: '#{newname}'\n"
				data.collect{|i| file.print "#{i}"}
			else
				print "couldn't open file for writeing\n"
				return false
			end
		when "Tempfile"
			begin
				if File.rename(data.path, "#{dir}/#{newname}")
					print "	Saving as: '#{newname}'\n"
				else
					print "couldn't rename tempfile\n"
					return false
				end
			rescue Errno::ENOENT
				print "Caught Errno::ENOENT (save_file)\n"
				print "Error: #{$!}\n"
				print "What the #@$$ happended?\n"
				return false
			end
		else
			print "EEEEPS Can't save data of type: #{data.type.to_s}\n"
			return false
	end
	return true
end

def parse_options(options)
	begin
		opts = GetoptLong.new(
			[ "-I", "--include",		GetoptLong::REQUIRED_ARGUMENT ],
			[ "-c", "--configfile",		GetoptLong::REQUIRED_ARGUMENT ],
			[ "-L", "--longname",		GetoptLong::NO_ARGUMENT ],
			[ "-C", "--combinedname",	GetoptLong::NO_ARGUMENT ],
			[ "-M", "--multipart",		GetoptLong::NO_ARGUMENT ],
			[ "-S", "--singlepart",		GetoptLong::NO_ARGUMENT ],
			[ "-T", "--test",		GetoptLong::NO_ARGUMENT ],
			[ "-X", "--exclude",		GetoptLong::REQUIRED_ARGUMENT ]
		)
		opts.quiet=true

		opts.each do |opt, arg|
			options[opt] = arg
		end
	rescue GetoptLong::InvalidOption
		print "#{$!}\n"
		usage
	end

	return options
end

def usage
	print "\nUsage:\n\n"
	print "ripnews.rb [-I <pattern>] [-c <file>] [-L] [-C] [-M] [-S] [-T] [-X <pattern>]\n\n"
	print "-I <pattern>	specify an include pattern\n"
	print "-c <file>	specify an alternate configfile\n"
	print "-L		use subject as filename\n"
	print "-C		use combined filenames\n"
	print "-M		get multipart articles\n"
	print "-S		get singlepart articles\n"
	print "-T		test mode, don't update newsrc file\n"
	print "-X <pattern>	specify an exclude pattern\n"
	exit
end

def parse_config(default = {})
	file = File.new("#{default[\"-c\"]}")
	lines = file.readlines

	i = 0
	group = ""
	grouparr = []
	@config = {}

	lines.collect!{|x|
		x.sub!(/^\s*/, "")
		x.sub!(/\#.*$/, "")
		x.sub!(/\s*$/, "")
		x.chomp
	}
	while i < lines.length
		line = lines[i]
		while line.sub!(/\s*\\$/, "") != nil
			line << lines[i+1]
			i += 1
		end
		line.sub!(/\s*$/, "")
		i += 1
		if line =~ /^OPT_(.*?)=(.*)/
			line = "-#{$1}=#{$2}"
		end
		print "#{i}: #{line}\n" if Debuglevel > 1
		if line =~ /(.*?)\s*\+=\s*(.*)/
			if group == ""
				if default.has_key?($1)
					default[$1] << $2
				else
					default[$1] = $2
				end
			else
				grouparr.collect{|g|
					if @config[g].has_key?($1)
						@config[g][$1] << $2
					elsif default.has_key?($1)
						@config[g][$1] = default[$1] + $2
					else
						@config[g][$1] = $2
					end
				}
			end
		elsif line =~ /(.*?)\s*=\s*(.*)/
			if group == ""
				default[$1] = $2
			else
				grouparr.collect{|g|
					@config[g][$1] = $2
				}
			end
		elsif line =~ /(.*?)\s*\{/
			group = $1
			grouparr = group.split('|')
			grouparr.collect{|g|
				@config[g] = {} unless @config.has_key?(g)
			}
		elsif line =~ /^}$/
			default.each_key{|x|
				grouparr.collect{|g|
					@config[g][x] = default[x] unless @config[g].has_key?(x)
				}
			}
			group = ""
			grouparr = []
		elsif line =~ /^$/
			next
		else
			print "Error parsing config on line: #{i}\n"
			exit
		end
	end

	if group != ""
		print "Error parsing config: group not terminated on line #{i}\n"
		exit
	end

	if Debuglevel > 2
		@config.each_key{|x|
			print "Group: #{x}\n"
			@config[x].each_key{|y|
				print "Key: '#{y}' => Value: '#{@config[x][y]}'\n"
			}
		}
	end
	return true
end

def check_config
	@config.each_key {|i|
		unless @config[i].has_key?("-I")
			print "No inclusions given for group #{i}. Won't match anything.\n"
			exit
		end
		@config[i]["DATADIR"] ="." unless @config[i].has_key?("DATADIR")
		@config[i]["PERMISSION"] = "0755" unless @config[i].has_key?("PERMISSION")
		if @config[i].has_key?("EXTENSIONS")
			@config[i]["-S"] = @config[i]["EXTENSIONS"]
			@config[i]["-M"] = @config[i]["EXTENSIONS"]
		end
		@config[i]["-M"] = "(?!.*)" if @config[i].has_key?("-S") and ! @config[i].has_key?("-M")
		@config[i]["-S"] = "(?!.*)" if @config[i].has_key?("-M") and ! @config[i].has_key?("-S")
	}
end

def get_single(subj)
	print "Fetching singlepart article: #{subj}\n"
	body = @articles.get_group_body(subj)
	if @articles.is_uuencoded(body)
		mode, filename, body = @articles.uudecode(body)
		return false unless check_ext(filename, "s")
		return mode, filename, body
	end
	if @articles.is_yencoded(body)
		mode, filename, body = @articles.ydecode(body)
		return false unless check_ext(filename, "s")
		return mode, filename, body
	end
	print "	Unknown encoding (not UU, not yEnc), skipping...\n"
	return false
end

def get_multi(subj, group)
	print "Fetching multipart article: #{subj}\n"
	if @config[group]["TEMPDIR"] == nil or @config[group]["TEMPDIR"] == ""
		body = @articles.get_group_body(subj)
		if @articles.is_uuencoded(body)
			mode, filename, body = @articles.uudecode(body)
			return false unless check_ext(filename, "m")
			return mode, filename, body
		elsif @articles.is_yencoded(body)
			mode, filename, body = @articles.ydecode(body)
			return false unless check_ext(filename, "m")
			return mode, filename, body
		end
		print "	Unknown encoding (not UU, not yEnc), skipping...\n"
		return false
	else
		body = @articles.get_group_body_first(subj)
		if @articles.is_uuencoded(body) or @articles.is_yencoded(body)
			file = Tempfile.new("riptmp", @config[group]["TEMPDIR"])
			body.collect{|x| file.print "#{x}\n"}
			return false unless @articles.get_group_body_rest(subj, file)
			fileout = Tempfile.new("riptmp", @config[group]["TEMPDIR"])
			if @articles.is_uuencoded(body)
				mode, filename, body = @articles.uudecode(file, fileout)
			elsif @articles.is_yencoded(body)
				mode, filename, body = @articles.ydecode(file, fileout)
			end
			return false unless check_ext(filename, "m")
			body = fileout.path
			bodybase = body.sub(/\/[^\/]*$/, "")
			File.rename(body, "#{bodybase}/ripnewsdecode")
			body = "#{bodybase}/ripnewsdecode"
			file.close
			fileout.close(false)
			return mode, filename, body
		else
			print "	Unknown encoding (not UU, not yEnc), skipping...\n"
			return false
		end
	end
end

def output_data(subject, mode, filename="", body="")
	group = @articles.get_groupname
	print "	mode: #{mode}\n" if Debuglevel > 0
	print "	Filename: '#{filename}'\n" if Debuglevel > 0
	if @config[group].has_key?("-L") and @config[group]["-L"]
		print "longname\n" if Debuglevel > 1
		outfile = subject
		while outfile.length > @maxfilelength
			outfile = outfile[0...-1]
		end
	elsif @config[group].has_key?("-C") and @config[group]["-C"]
		print "combinedname\n" if Debuglevel > 1
		outfile = "#{subject} [#{filename}]"
		sub2 = subject
		while outfile.length > @maxfilelength
			lastlength = outfile.length
			sub2 = sub2[0...-1]
			outfile = "#{sub2} [#{filename}]"	# this is going to loop if the #{filename} is too long :(
			if outfile.length == lastlength
				outfile = filename
				while outfile.length > @maxfilelength
					outfile = outfile[0...-1]
				end
			end
		end
	else
		print "shortname\n" if Debuglevel > 1
		outfile = filename
		while outfile.length > @maxfilelength
			outfile = outfile[0...-1]
		end
	end
	if save_file("#{@config[group]["DATADIR"]}/#{group}", outfile, body)
		@articles.group_update_newsrc(subject)
		@articles.save_newsrc unless @config[group].has_key?("-T") and @config[group]["-T"]
	end
end

def check_ext(filename, mode)
	case mode
	when "s"
		return @config.has_key?("-S") ? filename =~ /\.(#{@config["-S"]})$/ : true
	when "m"
		return @config.has_key?("-M") ? filename =~ /\.(#{@config["-M"]})$/ : true
	else
		print "Illegal mode \"#{mode}\" in check_ext\n"
		exit
	end
end

def get_max_file_length(tempdir=".")
i = 500
name = "a"*i
begin
	file = File.new("#{tempdir}/#{name}", "w", "0644").close
	File.delete("#{tempdir}/#{name}")
rescue Errno::ENAMETOOLONG
	i -= 1
	name = "a"*i
	retry
end
	i -= 14		# this is how many characters are still likely to be appended
			# is the filename already exists '-<#{date}.#{count}>' in save_file
			# this could be brought back to 5 '-<#{count}>' ...
	return i
end

#############################################################################################

$stdout.sync=true					# line buffered output
defaults = {'-c' => "#{ENV['HOME']}/.ripnewsrc"}
defaults = parse_options(defaults)
parse_config(defaults)
check_config

@maxfilelength = get_max_file_length(@config[@config.keys[0]]["TEMPDIR"])

print "$Id$\n"

if Debuglevel > 2
	@config.each_key{|i|
		print "Group: #{i}\n"
		@config[i].each_key{|j|
			print "Opt: #{j} val: #{@config[i][j]}\n"
		}
	}
end

for group in @config.keys.sort
	print "Getting articles for #{group}\n"
	@articles = Article.new(@config[group]["NNTPSERVER"], group, @config[group]["NEWSRCNAME"])
#	begin
		@articles.get_articles(@config[group]["CACHEDIR"])
#	rescue Article::
#		print "Caught something: #{$!}\n"
#		@articles.quit
#		next
#	end

	unless FileTest.directory?("#{@config[group]["DATADIR"]}/#{group}") or
			Dir.mkdir("#{@config[group]["DATADIR"]}/#{group}", @config[group]["PERMISSION"].oct)
		print "eeeps, couldn't create dir\n"
		exit
	end
	for i in @articles.get_group_subjects
		print "#{i}\n" if Debuglevel > 2
		if !(@config[group].has_key?("-X") and i =~ /#{@config[group]["-X"]}/) and
		   i =~ /#{@config[group]["-I"]}/
			print "Match: #{i}\n" if Debuglevel > 0
			if @articles.group_is_complete(i)
				begin
					if @articles.group_is_singlepart(i)
						mode, filename, body = get_single(i)
					elsif @articles.group_is_multipart(i)
						mode, filename, body = get_multi(i, group)
					end
					output_data(i, mode, filename, body) if mode != false
				rescue Article::TempError, Article::PermError
					print "#{$!}\n"
					print "Skipping article...\n"
					next
				end
			else
				print "	Not complete: #{i}\n"
			end
		end
	end
	@articles.quit
end