Index: branches/1.0/jobarchived/jobarchived.py
===================================================================
--- branches/1.0/jobarchived/jobarchived.py	(revision 855)
+++ branches/1.0/jobarchived/jobarchived.py	(revision 857)
@@ -568,5 +568,7 @@
         return self.addJob( jobid, jobattrs )
 
-    def checkStaleJobs( self ):
+    def checkTimedoutJobs( self ):
+
+        debug_msg( 1, 'Housekeeping: checking database for timed out jobs..' )
 
         # Locate all jobs in the database that are not set to finished
@@ -580,25 +582,20 @@
             return None
 
-        cleanjobs    = [ ]
-        timeoutjobs    = [ ]
-
-        jobtimeout_sec    = JOB_TIMEOUT * (60 * 60)
-        cur_time    = time.time()
+        timeoutjobs  = [ ]
+
+        jobtimeout_sec = JOB_TIMEOUT * (60 * 60)
+        cur_time       = time.time()
 
         for row in r:
 
-            job_id            = row[0]
-            job_requested_time    = row[4]
-            job_status        = row[7]
-            job_start_timestamp    = row[8]
+            job_id              = row[0]
+            job_requested_time  = row[4]
+            job_status          = row[7]
+            job_start_timestamp = row[8]
 
             # If it was set to queued and we didn't see it started
             # there's not point in keeping it around
             #
-            if job_status == 'Q' or not job_start_timestamp:
-
-                cleanjobs.append( job_id )
-
-            else:
+            if job_status == 'R' and job_start_timestamp:
 
                 start_timestamp = int( job_start_timestamp )
@@ -617,14 +614,7 @@
                     timeoutjobs.append( (job_id, job_start_timestamp, rtime_epoch) )
 
-        debug_msg( 1, 'Found ' + str( len( cleanjobs ) ) + ' stale jobs in database: deleting entries' )
-
-        # Purge these from database
-        #
-        for j in cleanjobs:
-
-            q = "DELETE FROM jobs WHERE job_id = '" + str( j ) + "'"
-            self.setDatabase( q )
-
-        debug_msg( 1, 'Found ' + str( len( timeoutjobs ) ) + ' timed out jobs in database: closing entries' )
+        debug_msg( 1, 'Housekeeping: Found ' + str( len( timeoutjobs ) ) + ' timed out jobs in database: closing entries' )
+
+        ret_jobids_clean = [ ]
 
         # Close these jobs in the database
@@ -639,6 +629,62 @@
                 new_end_timestamp    = int( s ) + r
 
-            q = "UPDATE jobs SET job_status='F',job_stop_timestamp = '" + str( new_end_timestamp ) + "' WHERE job_id = '" + str(i) + "'"
+                q = "UPDATE jobs SET job_status='F',job_stop_timestamp = '" + str( new_end_timestamp ) + "' WHERE job_id = '" + str(i) + "'"
+                self.setDatabase( q )
+            else:
+
+                # Requested walltime unknown: cannot guess end time: sorry delete them
+                q = "DELETE FROM jobs WHERE job_id = '" + str( i ) + "'"
+                self.setDatabase( q )
+
+            ret_jobids_clean.append( i )
+
+        debug_msg( 1, 'Housekeeping: done.' )
+
+        return ret_jobids_clean
+
+    def checkStaleJobs( self ):
+
+        debug_msg( 1, 'Housekeeping: checking database for stale jobs..' )
+
+        # Locate all jobs in the database that are not set to finished
+        #
+        q = "SELECT * from jobs WHERE job_status != 'F'"
+
+        r = self.getDatabase( q )
+
+        if len( r ) == 0:
+
+            return None
+
+        cleanjobs      = [ ]
+
+        cur_time       = time.time()
+
+        for row in r:
+
+            job_id              = row[0]
+            job_requested_time  = row[4]
+            job_status          = row[7]
+            job_start_timestamp = row[8]
+
+            # If it was set to queued and we didn't see it started
+            # there's not point in keeping it around
+            #
+            if job_status == 'Q' or not job_start_timestamp:
+
+                cleanjobs.append( job_id )
+
+        debug_msg( 1, 'Housekeeping: Found ' + str( len( cleanjobs ) ) + ' stale jobs in database: deleting entries' )
+
+        # Purge these from database
+        #
+        for j in cleanjobs:
+
+            q = "DELETE FROM jobs WHERE job_id = '" + str( j ) + "'"
             self.setDatabase( q )
+
+        debug_msg( 1, 'Housekeeping: done.' )
+
+        return cleanjobs
 
 class RRDMutator:
@@ -848,4 +894,9 @@
         self.jobAttrsSaved   = { }
 
+        self.iteration       = 0
+
+        self.ds.checkTimedoutJobs()
+        self.ds.checkStaleJobs()
+
         debug_msg( 1, "XML: Handler created" )
 
@@ -855,6 +906,17 @@
         self.heartbeat      = 0
         self.elementct      = 0
-
-        debug_msg( 1, "XML: Start document" )
+        self.iteration      = self.iteration + 1
+
+        if self.iteration > 20:
+
+            timedout_jobs = self.ds.checkTimedoutJobs()
+            self.iteration = 0
+
+            for j in timedout_jobs:
+
+                del self.jobAttrs[ j ]
+                del self.jobAttrsSaved[ j ]
+
+        debug_msg( 1, "XML: Start document: iteration %s" %str(self.iteration) )
 
     def startElement( self, name, attrs ):
@@ -1066,19 +1128,11 @@
         """Setup initial variables and gather info on existing rrd archive"""
 
-        self.config    = config
-        self.clusters    = { }
-        self.ds        = datastore
-
-        debug_msg( 1, 'Checking database..' )
-
-        global DEBUG_LEVEL
-
-        if DEBUG_LEVEL <= 2:
-            self.ds.checkStaleJobs()
-
-        debug_msg( 1, 'Check done.' )
-        debug_msg( 1, 'Checking rrd archive..' )
+        self.config          = config
+        self.clusters        = { }
+        self.ds              = datastore
+
+        debug_msg( 1, 'Housekeeping: checking RRD archive (may take a while)..' )
         self.gatherClusters()
-        debug_msg( 1, 'Check done.' )
+        debug_msg( 1, 'Housekeeping: RRD check complete.' )
 
     def gatherClusters( self ):
@@ -1426,5 +1480,5 @@
             STORE_INTERVAL = 60
         else:
-            STORE_INTERVAL = random.randint( 360, 640 )
+            STORE_INTERVAL = random.randint( 300, 600 )
 
         try:
