Changeset fe95b7 for src

Timestamp:

Jun 29, 2012, 8:01:38 AM (13 years ago)

Author:

Frederik Heber <heber@…>

Branches:

Action_Thermostats, Add_AtomRandomPerturbation, Add_FitFragmentPartialChargesAction, Add_RotateAroundBondAction, Add_SelectAtomByNameAction, Added_ParseSaveFragmentResults, AddingActions_SaveParseParticleParameters, Adding_Graph_to_ChangeBondActions, Adding_MD_integration_tests, Adding_ParticleName_to_Atom, Adding_StructOpt_integration_tests, AtomFragments, Automaking_mpqc_open, AutomationFragmentation_failures, Candidate_v1.5.4, Candidate_v1.6.0, Candidate_v1.6.1, ChangeBugEmailaddress, ChangingTestPorts, ChemicalSpaceEvaluator, CombiningParticlePotentialParsing, Combining_Subpackages, Debian_Package_split, Debian_package_split_molecuildergui_only, Disabling_MemDebug, Docu_Python_wait, EmpiricalPotential_contain_HomologyGraph, EmpiricalPotential_contain_HomologyGraph_documentation, Enable_parallel_make_install, Enhance_userguide, Enhanced_StructuralOptimization, Enhanced_StructuralOptimization_continued, Example_ManyWaysToTranslateAtom, Exclude_Hydrogens_annealWithBondGraph, FitPartialCharges_GlobalError, Fix_BoundInBox_CenterInBox_MoleculeActions, Fix_ChargeSampling_PBC, Fix_ChronosMutex, Fix_FitPartialCharges, Fix_FitPotential_needs_atomicnumbers, Fix_ForceAnnealing, Fix_IndependentFragmentGrids, Fix_ParseParticles, Fix_ParseParticles_split_forward_backward_Actions, Fix_PopActions, Fix_QtFragmentList_sorted_selection, Fix_Restrictedkeyset_FragmentMolecule, Fix_StatusMsg, Fix_StepWorldTime_single_argument, Fix_Verbose_Codepatterns, Fix_fitting_potentials, Fixes, ForceAnnealing_goodresults, ForceAnnealing_oldresults, ForceAnnealing_tocheck, ForceAnnealing_with_BondGraph, ForceAnnealing_with_BondGraph_continued, ForceAnnealing_with_BondGraph_continued_betteresults, ForceAnnealing_with_BondGraph_contraction-expansion, FragmentAction_writes_AtomFragments, FragmentMolecule_checks_bonddegrees, GeometryObjects, Gui_Fixes, Gui_displays_atomic_force_velocity, ImplicitCharges, IndependentFragmentGrids, IndependentFragmentGrids_IndividualZeroInstances, IndependentFragmentGrids_IntegrationTest, IndependentFragmentGrids_Sole_NN_Calculation, JobMarket_RobustOnKillsSegFaults, JobMarket_StableWorkerPool, JobMarket_unresolvable_hostname_fix, MoreRobust_FragmentAutomation, ODR_violation_mpqc_open, PartialCharges_OrthogonalSummation, PdbParser_setsAtomName, PythonUI_with_named_parameters, QtGui_reactivate_TimeChanged_changes, Recreated_GuiChecks, Rewrite_FitPartialCharges, RotateToPrincipalAxisSystem_UndoRedo, SaturateAtoms_findBestMatching, SaturateAtoms_singleDegree, StoppableMakroAction, Subpackage_CodePatterns, Subpackage_JobMarket, Subpackage_LinearAlgebra, Subpackage_levmar, Subpackage_mpqc_open, Subpackage_vmg, Switchable_LogView, ThirdParty_MPQC_rebuilt_buildsystem, TrajectoryDependenant_MaxOrder, TremoloParser_IncreasedPrecision, TremoloParser_MultipleTimesteps, TremoloParser_setsAtomName, Ubuntu_1604_changes, stable

Children:

f98c8e

Parents:

559385

git-author:

Frederik Heber <heber@…> (03/04/12 16:18:32)

git-committer:

Frederik Heber <heber@…> (06/29/12 08:01:38)

Message:

FragmentQueue can now resubmit jobs.

• we keep an internal list of jobs currently being worked on. If the worker returns with a failure, they can be resubmit to the queue.
• there are Max_Attempts (currently set to 1) tries to resubmit.
• added unit test function on this.
• added regression test Fragmentation/Automation resubmitjobs.

Location:

src/Fragmentation/Automation

Files:

• FragmentQueue.cpp (modified) (3 diffs)
• FragmentQueue.hpp (modified) (2 diffs)
• unittests/FragmentQueueUnitTest.cpp (modified) (1 diff)
• unittests/FragmentQueueUnitTest.hpp (modified) (2 diffs)

src/Fragmentation/Automation/FragmentQueue.cpp

@@ -23 +23 @@
 
 #include "CodePatterns/Assert.hpp"
+#include "CodePatterns/Log.hpp"
 
 FragmentResult::ptr FragmentQueue::NoResult( new FragmentResult(-1) );
 FragmentResult::ptr FragmentQueue::NoResultQueued( new FragmentResult(-2) );
 FragmentResult::ptr FragmentQueue::ResultDelivered( new FragmentResult(-3) );
+size_t FragmentQueue::Max_Attempts = (size_t)1;
 
 /** Constructor for class FragmentQueue.
@@ -93 +95 @@
       "FragmentQueue::popJob() - there are no jobs on the queue.");
   FragmentJob::ptr job = jobs.front();
+#ifndef NDEBUG
+  std::pair< BackupMap::iterator, bool> inserter =
+#endif
+  backup.insert( std::make_pair( job->getId(), job ));
+  ASSERT (inserter.second,
+      "FragmentQueue::popJob() - job "+toString(job->getId())+
+      " is already in the backup.");
   ResultMap::iterator iter = results.find(job->getId());
   ASSERT(iter != results.end(),
@@ -197 +206 @@
 void FragmentQueue::pushResult(FragmentResult::ptr &_result)
 {
+  const JobId_t id = _result->getId();
   /// check for presence
-  ResultMap::iterator iter = results.find(_result->getId());
+  ResultMap::iterator iter = results.find(id);
   ASSERT(iter != results.end(),
-      "FragmentQueue::pushResult() - job "+toString(_result->getId())+" is not known to us.");
+      "FragmentQueue::pushResult() - job "+toString(id)+" is not known to us.");
   ASSERT(*iter->second == *NoResultQueued,
-      "FragmentQueue::pushResult() - is not waiting for the result of job "+toString(_result->getId())+".");
-  /// and overwrite NoResult in found entry
-  iter->second = _result;
-}
-
+      "FragmentQueue::pushResult() - is not waiting for the result of job "+toString(id)+".");
+  // check whether this is a resubmitted job
+  AttemptsMap::iterator attemptiter = attempts.find(id);
+  // check whether succeeded or (finally) failed
+  if ((_result->exitflag == 0) || ((attemptiter != attempts.end()) && (attemptiter->second >= Max_Attempts))) {
+    // give notice if it is resubmitted job
+    if (attemptiter != attempts.end()) {
+      if (attemptiter->second >= Max_Attempts)
+        ELOG(1, "Job #" << id << " failed on " << Max_Attempts << "th attempt for the last time.");
+      else
+        LOG(1, "INFO: Job #" << id << " succeeded on " << attemptiter->second << "th attempt.");
+    }
+    // remove in attempts
+    if (attemptiter != attempts.end())
+      attempts.erase(attemptiter);
+    // remove in backup map
+    BackupMap::iterator backupiter = backup.find(id);
+    ASSERT( backupiter != backup.end(),
+        "FragmentQueue::pushResult() - cannot find job "+toString(id)
+        +" in backup.");
+    backup.erase(backupiter);
+    /// and overwrite NoResult in found entry
+    iter->second = _result;
+  } else {
+    LOG(1, "Job " << id << " failed, resubmitting.");
+    // increase attempts
+    if (attemptiter != attempts.end())
+      ++(attemptiter->second);
+    else
+      attempts.insert( std::make_pair(id, (size_t)1) );
+    // resubmit job
+    resubmitJob(id);
+  }
+}
+
+/** Resubmit a job which a worker failed to calculate.
+ *
+ * @param jobid id of the failed job
+ */
+void FragmentQueue::resubmitJob(const JobId_t jobid)
+{
+  BackupMap::iterator iter = backup.find(jobid);
+  ASSERT( iter != backup.end(),
+      "FragmentQueue::resubmitJob() - job id "+toString(jobid)
+      +" not stored in backup.");
+  if (iter != backup.end()) {
+    // remove result
+    ResultMap::iterator resiter = results.find(jobid);
+    ASSERT( resiter != results.end(),
+        "FragmentQueue::resubmitJob() - resubmitting job "+toString(jobid)
+        +" for which no result is present.");
+    results.erase(resiter);
+    pushJob(iter->second);
+    backup.erase(iter);
+  }
+}
+

src/Fragmentation/Automation/FragmentQueue.hpp

@@ -40 +40 @@
   FragmentJob::ptr popJob();
   bool isJobPresent() const;
+  void resubmitJob(const JobId_t jobid);
 
   // querying for results
@@ -59 +60 @@
   //!> result that takes place in ResultQueue after real result has been delivered.
   static FragmentResult::ptr ResultDelivered;
+  //!> maximum number of attempts before job is not resubmitted
+  static size_t Max_Attempts;
 
   typedef std::deque<FragmentJob::ptr> JobQueue;
+  typedef std::map<JobId_t, FragmentJob::ptr> BackupMap;
   typedef std::map<JobId_t, FragmentResult::ptr> ResultMap;
+  typedef std::map<JobId_t, size_t> AttemptsMap;
 
   //!> queue for all jobs
   JobQueue jobs;
+  //!> backup map for popped jobs that workers failed to calculate
+  BackupMap backup;
   //!> map for all results that have been polled by the server
   ResultMap results;
+  //!> map for all jobs that had to be resubmitted storing how often this has been tried
+  AttemptsMap attempts;
 };
 

src/Fragmentation/Automation/unittests/FragmentQueueUnitTest.cpp

@@ -294 +294 @@
 }
 
+/** Unit test for resubmitJob().
+ *
+ */
+void FragmentQueueTest::resubmitTest()
+{
+  FragmentJob::ptr testJob(new FragmentJobStub(1));
+
+  // push a Job into queue
+#ifndef NDEBUG
+  CPPUNIT_ASSERT_NO_THROW( queue->pushJob(testJob) );
+#else
+  queue->pushJob(testJob);
+#endif
+  CPPUNIT_ASSERT_EQUAL((size_t)1, queue->jobs.size());
+  CPPUNIT_ASSERT_EQUAL((size_t)1, queue->results.size());
+
+  // pop the job
+  // pop both as if some work was being done
+#ifndef NDEBUG
+  CPPUNIT_ASSERT_NO_THROW( queue->popJob() );
+#else
+  queue->popJob();
+#endif
+  CPPUNIT_ASSERT_EQUAL((size_t)0, queue->jobs.size());
+  CPPUNIT_ASSERT_EQUAL((size_t)1, queue->results.size());
+
+  // resubmit
+#ifndef NDEBUG
+  CPPUNIT_ASSERT_NO_THROW( queue->resubmitJob((JobId_t)1) );
+#else
+  queue->resubmitJob((JobId_t)1);
+#endif
+
+  // check whethers it's present again
+  CPPUNIT_ASSERT_EQUAL((size_t)1, queue->jobs.size());
+  CPPUNIT_ASSERT_EQUAL((size_t)1, queue->results.size());
+}
+

src/Fragmentation/Automation/unittests/FragmentQueueUnitTest.hpp

@@ -28 +28 @@
     CPPUNIT_TEST ( ResultsTest );
     CPPUNIT_TEST ( AllResultsTest );
+    CPPUNIT_TEST ( resubmitTest );
     CPPUNIT_TEST_SUITE_END();
 
@@ -37 +38 @@
       void ResultsTest();
       void AllResultsTest();
+      void resubmitTest();
 
 private: