[rt-users] time delay in the queue
Travis Campbell
travis.campbell at amd.com
Fri Jan 17 19:12:52 EST 2003
[Apologies if this gets double posted. My fingers had a moment of their
own before I was able to finish my message :-)]
On Fri, Jan 17, 2003 at 03:23:21PM -0800, Anshuman Kanwar wrote:
> Thanks for the reply Seth.
>
> >From what I could gather this runs a nightly check and increments the
> priorities. Right ? I am looking for something that escalates in the order
> of 15-20 minutes. Maybe I should roll my own ...
I built an escalator meant to run every five minutes from cron for monitoring
tickets coming into our crisis queue.
We have a site policy of responding to these tickets within fifteen minutes,
otherwise it starts adding people on as watchers (which happen to be our
email addresses for our pagers). Every fifteen minutes, if no one has done
something with the ticket, another person (or level of management) gets
tacked on, priority gets bumped, and all watchers are notified. This
continues until someone moves it out of the queue or resolves it.
This queue is specifically configured so that priority starts at 1 and goes
to 99 in one day (because the escalator runs so often).
The queue is also setup to NotifyAdminCc's on Correspondance and on Comment
so we don't have to build any sort of notifications into the script; RT does
it for us. We use a template that only sends a minimal amount of info (queue
and ticket number) because it's being sent to an alphanumeric pager with a
limited character count for incoming messages.
The only caveat with we found is that if you do this for a specific queue,
and have another escalator that does a daily priority bump for all your
queues, you need to make sure both escalations don't collide. We got around
this by doing something like
$tickets->LimitQueue(VALUE => 'MyQueue', OPERATOR => '!=');
in our daily escalator.
I've attached it. YMMV.
(I hope this makes sense. I'm being bombarded by the kids while I write
this 8-)
Travis
--
Travis Campbell - Unix Systems Administrator = travis at mpdtxmail.amd.com
5900 E. Ben White Blvd, Austin, TX 78741 = travis.campbell at amd.com
TEL: (512) 602-1888 PAG: (512) 604-0341 = webmaster at mpdtxmail.amd.com
=============================================================================
"Does anything work as expected?" Yes. An axe through the CPU.
-------------- next part --------------
#!/opt/mpd/bin/perl -w
#
use strict;
use Carp;
use lib "/opt/rt2/lib";
use lib "/opt/rt2/etc";
use IO::File;
use Mail::Mailer;
use Proc::ProcessTable;
my $lock = "/var/run/crisis-escalator.pid";
my $DEBUG = undef;
GetLock($lock);
use RT::Interface::CLI qw(CleanEnv LoadConfig DBConnect
GetCurrentUser GetMessageContent);
print "Loading environment\n" if ($DEBUG);
#Clean out all the nasties from the environment
CleanEnv();
#Load etc/config.pm and drop privs
LoadConfig();
#Connect to the database and get RT::SystemUser and RT::Nobody loaded
DBConnect();
use RT::Date;
use RT::Tickets;
my $now = new RT::Date($RT::Nobody);
$now->SetToNow();
my $first_level;
my $second_level;
my ($sec,$min,$hour,$wday) = (gmtime())[0,1,2,6];
$hour = $hour - 5;
#
# Determine what time it is so we send an alert to the people for first and
# second level notifications
#
if (($hour <= 9 and $min <= 59) and
($hour >= 17 and $min >= 5) and
($wday > 0 and $wday < 6)) {
$first_level = 'admin-pager at mail.example.com'; # on call pager
$second_level = 'eod-pager at mail.example.com'; # engineer of the day
} else {
$first_level = 'eod-pager at mail.example.com'; # engineer of the day
$second_level = 'admin-pager at mail.example.com';# on call pager
}
my $tickets = new RT::Tickets($RT::SystemUser);
$tickets->LimitQueue(VALUE => 'Crisis-TX');
$tickets->LimitStatus(VALUE => 'open');
$tickets->LimitStatus(VALUE => 'new');
while (my $ticket = $tickets->Next) {
print "Working on tickets.\n" if ($DEBUG);
my $priority = $ticket->Priority || 1;
if ($priority == 1) {
print "First alert.\n" if ($DEBUG);
$ticket->AddAdminCc(Email => $first_level);
TicketComment($ticket, $priority);
PrioBump($ticket);
BailOut($lock);
} elsif ($priority == 2) {
print "Second alert. After 15 minutes.\n" if ($DEBUG);
if (CheckAge($ticket, $now, 900)) {
TicketComment($ticket, $priority);
PrioBump($ticket);
} else {
BailOut($lock);
}
} elsif ($priority == 3) {
print "Third alert. After 30 minutes.\n" if ($DEBUG);
if (CheckAge($ticket, $now, 1800)) {
$ticket->AddAdminCc(Email => $second_level);
TicketComment($ticket, $priority);
PrioBump($ticket);
} else {
BailOut($lock);
}
} elsif ($priority == 4) {
print "Fourth alert. After 45 minutes.\n" if ($DEBUG);
if (CheckAge($ticket, $now, 2700)) {
$ticket->AddAdminCc(Email => 'boss-pager at mail.example.com');
TicketComment($ticket, $priority);
PrioBump($ticket);
} else {
BailOut($lock);
}
} elsif ($priority == 5) {
print "Fifth alert. After 60 minutes.\n" if ($DEBUG);
if (CheckAge($ticket, $now, 3600)) {
TicketComment($ticket, $priority);
PrioBump($ticket);
} else {
BailOut($lock);
}
} elsif ($priority >= 6 and ($priority % 4 + 1) == 1) {
print "Hourly alert.\n" if ($DEBUG);
if (CheckAge($ticket, $now, 3600, -1)) {
TicketComment($ticket, $priority);
PrioBump($ticket);
} else {
BailOut($lock);
}
} elsif ($priority == 99) {
print "Bad alert.\n" if ($DEBUG);
if (CheckAge($ticket, $now, 900)) {
$ticket->AddAdminCc(Email => 'team-pager at mail.example.com');
TicketComment($ticket, $priority);
} else {
BailOut($lock);
}
}
}
BailOut($lock);
sub PrioBump {
my $self = shift;
unless ($self->Priority == $self->FinalPriority() ) {
print "Evaluating ".$self->Id. "..." if ($DEBUG);
print $self->Priority." -> " if ($DEBUG);
$self->SetPriority($self->Priority + 1);
print $self->Priority."\n" if ($DEBUG);
}
}
sub TicketComment {
my $self = shift;
my $prio = shift;
require MIME::Entity;
my $comment;
if ($prio == 1) {
$comment = 'New ticket. First alert being sent.';
} else {
$comment = 'No input from Systeam. Ticket escalated to next level and notifications are being sent.';
}
my $Comment = MIME::Entity->build(Data => $comment);
$self->Comment(MIMEObj => $Comment);
}
sub CheckAge {
my $ticket = shift;
my $now = shift;
my $age = shift;
my $created = $ticket->CreatedObj->Unix;
my $seconds = $now->Diff($created, $now);
if ($seconds <= $age) {
return 0;
}
return 1;
}
sub BailOut {
my $lock = shift;
print "Bailing out.\n" if ($DEBUG);
$RT::Handle->Disconnect();
ClearLock($lock);
exit;
}
sub GetLock {
my $lock = shift;
my $now = time;
my $pid = $$;
if ( -f $lock ) {
my $mtime = (stat($lock))[9];
my $oldpid = FindPid($lock,$pid);
if ($oldpid) {
print "Previous escalator still running ($oldpid) ($now)\n" if ($DEBUG);
exit;
} else {
print "Clearing lock from previous escalator ($oldpid)\n" if ($DEBUG);
ClearLock($lock);
}
}
if ( ! -f $lock ) {
print "Creating lock file for $pid\n" if ($DEBUG);
my $fh = IO::File->new(">$lock") or
die "Could not write to lock file ($pid): $!";
print $fh $pid, "\n";
$fh->close;
}
}
sub FindPid {
my $lock = shift;
my $pid = shift;
my $fh = IO::File->new("<$lock") or
die "Lock file exists but I couldn't open it for reading: $!";
my $runningpid = <$fh>;
chomp $runningpid;
$fh->close;
my $p = Proc::ProcessTable->new();
my $pref = $p->table;
for my $process (@$pref) {
if ($process->uid == $runningpid) {
return $process->uid;
}
}
return 0;
}
sub ClearLock {
my $lock = shift;
print "Clearing lockfile $lock\n" if ($DEBUG);
unlink $lock or
die "Could not unlink $lock: $!";
}
More information about the rt-users
mailing list