From 7d3679111860ea270074bff4699b51a16a88bea2 Mon Sep 17 00:00:00 2001 From: Ward Wouts Date: Mon, 30 Nov 2009 15:59:57 +0000 Subject: [PATCH] sync met werkbranch --- pbatch/nmap-tcp-1000-add | 2 +- pbatch/nmap-udp-1000-add | 2 +- pbatch/pb.rb | 145 ++++++++++++++++++++------------------- pbatch/stderrandout.rb | 12 ++++ 4 files changed, 90 insertions(+), 71 deletions(-) create mode 100755 pbatch/stderrandout.rb diff --git a/pbatch/nmap-tcp-1000-add b/pbatch/nmap-tcp-1000-add index aca95a6..baa1a5f 100755 --- a/pbatch/nmap-tcp-1000-add +++ b/pbatch/nmap-tcp-1000-add @@ -1,4 +1,4 @@ #!/usr/bin/env bash for i in $(cat targets);do - ./pb add nmap -PN -v -r -sS -sV --version-light --reason -O -T4 -oN nmap-sS.${i}.txt ${i} + ./pb.rb add nmap -PN -v -r -sS -sV --version-light --reason -O -T4 -oN nmap-sS.${i}.txt ${i} done diff --git a/pbatch/nmap-udp-1000-add b/pbatch/nmap-udp-1000-add index c7ac0eb..478051e 100755 --- a/pbatch/nmap-udp-1000-add +++ b/pbatch/nmap-udp-1000-add @@ -1,4 +1,4 @@ #!/usr/bin/env bash for i in $(cat targets);do - ./pb add nmap -PN -v -r -sU -sV --version-light --reason -T4 -oN nmap-sU.${i}.txt ${i} + ./pb.rb add nmap -PN -v -r -sU -sV --version-light --reason -T4 -oN nmap-sU.${i}.txt ${i} done diff --git a/pbatch/pb.rb b/pbatch/pb.rb index 8b0e602..e88eb33 100755 --- a/pbatch/pb.rb +++ b/pbatch/pb.rb @@ -4,6 +4,8 @@ # $URL$ require 'fileutils' +require 'thread' +require 'open3' JOBSDIR = "jobs" PARKDIR = "park" @@ -14,6 +16,8 @@ MONITORDIR = "monitor" RUNMODE = 0755 CONFIG = "config" @config = { "waittime" => 3, "maxproc" => 3 } +@processor_threads = [] +@jobclaimlock = Mutex.new COUNTERFILE="jobscounter" RUNNINGFILE="runningcounter" @@ -22,19 +26,19 @@ def usage puts < -add >-->-------Add the given command as a job. -addfile|af >------Add the given file as a job. -createconfing>-->-------Write a default configuration file. -run>---->------->-------Start the master scheduler process. -del >---->-------Delete the given job from the queue. -help|-h>>------->-------Show this help. -park>--->------->-------Place all queued jobs in the parkinglot. -unpark>->------->-------Place all jobs from the parkinglot back in the queue. -retry >-->-------Reschedule a failed job. Destroys all output. -prio >---Change the priority of the given job.>--[unimplemented] -status|st>------>-------Show the status of the current jobs. -stop >--->-------Stop the given job.>----[unimplemented] -stopall>>------->-------Stop all running jobs.>-[unimplemented] +add Add the given command as a job. +addfile|af Add the given file as a job. +createconfing Write a default configuration file. +run Start the master scheduler process. +del Delete the given job from the queue. +help|-h Show this help. +park Place all queued jobs in the parkinglot. +unpark Place all jobs from the parkinglot back in the queue. +retry Reschedule a failed job. Destroys all output. +prio Change the priority of the given job. [unimplemented] +status|st Show the status of the current jobs. +stop Stop the given job. [unimplemented] +stopall Stop all running jobs. [unimplemented] ENDTXT exit end @@ -136,7 +140,7 @@ def addjobfromfile end jobnumber = sprintf("%06d", increasejobscounter) puts "New job: #{jobnumber}" - File.copy(filename, "#{@basedir}/#{JOBSDIR}/#{jobnumber}") + FileUtils.cp(filename, "#{@basedir}/#{JOBSDIR}/#{jobnumber}") File.chmod(RUNMODE, "#{@basedir}/#{JOBSDIR}/#{jobnumber}") end @@ -209,35 +213,6 @@ def changeprio puts "unimplemented" end -def incrementrunning - # als ik threading ga gebruiken is dit niet meer nodig (want geen runningfile nodig) - File.new(RUNNINGFILE).flock(File::LOCK_EX) - File.open(RUNNINGFILE, "r+"){|file| - running = file.read - running = running.to_i + 1 - file.seek(0) - file.trunc - file.puts running - } - File.new(RUNNINGFILE).flock(File::LOCK_UN) -end - -def decrementrunning - # als ik threading ga gebruiken is dit niet meer nodig (want geen runningfile nodig) - File.new(RUNNINGFILE).flock(File::LOCK_EX) - File.open(RUNNINGFILE, "r+"){|file| - running = file.read - running = running.to_i - 1 - if running < 0 - running = 0 - end - file.seek(0) - file.trunc - file.puts running - } - File.new(RUNNINGFILE).flock(File::LOCK_UN) -end - def getjob Dir.entries("#{@basedir}/#{JOBSDIR}").each{|file| if file == '.' or file == '..' @@ -250,6 +225,7 @@ def getjob end end } + return nil end def failjob(jobnumber) @@ -261,24 +237,46 @@ def finishjob(jobnumber) end def runprocessor - incrementrunning - sleep 1 # XXX niet nodig als locking echt werkt (of bij threading) - runjob = getjob - if ! runjob.nil? - puts "Starting job: #{runjob}" - owd = Dir.pwd - Dir.cd("#{@basedir}/#{PROCDIR}/#{runjob}") -# XXX IPC nog even uitzoeken... popen3 ? -# if "${basedir}/processors/${runjob}/job" 2>&1 > "${basedir}/processors/${runjob}/output" ;then -# finishjob ${runjob} - puts "Finished job: #{runjob}" -# else -# failjob ${runjob} - puts "Failed job: #{runjob}" -# fi - Dir.cd(owd) + @processor_threads << Thread.new do + begin + # gebruik een mutex voor het claimen van een job! + runjob = nil + @jobclaimlock.synchronize { + runjob = getjob + } + + if ! runjob.nil? + starttime = Time.now + puts "Starting job: #{runjob}" + owd = Dir.pwd +# puts "#{@basedir}/#{PROCDIR}/#{runjob}" + Dir.chdir("#{@basedir}/#{PROCDIR}/#{runjob}") + begin + output = File.open("#{@basedir}/#{PROCDIR}/#{runjob}/output", "w") + Open3.popen3("#{@basedir}/#{PROCDIR}/#{runjob}/job", "r+"){ |stdin, stdout, stderr| + # dit moet netter kunnen, nu wordt de volgorde van de diverse outputs veranderd + while !stdout.eof and ! stderr.eof do + #output.print "XXX ERR #{stderr.gets}" + #output.print "XXX OUT #{stdout.gets}" + output.print stderr.gets + output.print stdout.gets + end + } + output.close + finishjob runjob + puts "Finished job: #{runjob} [runtime: #{Time.now-starttime}]" + rescue + failjob runjob + puts "Failed job: #{runjob} [runtime: #{Time.now-starttime}]" + puts $! + end + Dir.chdir(owd) + end + rescue + puts "failure in thread" + puts $! + end end - decrementrunning end def stop @@ -291,18 +289,27 @@ end def master if Process.euid != 0 - puts "If your jobs need root priviledges (XXX spelling) you want to run this as root" + puts "If your jobs need root privileges you want to run this as root" end - # als ik threading ga gebruiken is dit niet meer nodig (want geen runningfile nodig) - if ! FileTest.file?(RUNNINGFILE) - FileUtils.touch(RUNNINGFILE) + while true + sleep @config["waittime"] +# puts "Number of threads: #{@processor_threads.length}" + # dode threads moeten opgeruimd worden + (0...@processor_threads.length).to_a.reverse.each{|thr| +# puts "thr: #{thr}" +# puts @processor_threads[thr].status + if @processor_threads[thr].status == false +# puts "deleting thread #{thr}" + @processor_threads.delete_at(thr) + end + } + if @processor_threads.length < @config["maxproc"] + # nog testen op aanwezig zijn van nieuwe jobs in queue? + runprocessor + end + readconfig end - File.new(RUNNINGFILE).flock(File::LOCK_EX) - File.open(RUNNINGFILE, "w"){|file| - file.puts "0" - } - File.new(RUNNINGFILE).flock(File::LOCK_UN) # while sleep $waittime; do # if [ $(cat "${basedir}/${runningfile}") -lt $maxproc ]; then diff --git a/pbatch/stderrandout.rb b/pbatch/stderrandout.rb new file mode 100755 index 0000000..4eae8a6 --- /dev/null +++ b/pbatch/stderrandout.rb @@ -0,0 +1,12 @@ +#!/usr/bin/env ruby + +$stderr.puts "STANDARD ERROR 1" +$stderr.puts "STANDARD ERROR 2" +$stderr.puts "STANDARD ERROR 3" +$stderr.puts "STANDARD ERROR 4" +$stderr.puts "STANDARD ERROR 5" +$stdout.puts "STANDARD OUT 1" +$stdout.puts "STANDARD OUT 2" +$stdout.puts "STANDARD OUT 3" +$stdout.puts "STANDARD OUT 4" +$stdout.puts "STANDARD OUT 5"