| 1 | ### fragment the molecule and check the number of configs
|
|---|
| 2 |
|
|---|
| 3 | AT_SETUP([Fragmentation - Automation: Resubmitting failed jobs])
|
|---|
| 4 | AT_KEYWORDS([fragmentation automation server worker jobadder])
|
|---|
| 5 |
|
|---|
| 6 | # check that ports are unique over all tests such that they may run in parallel
|
|---|
| 7 | WORKERPORT=11070
|
|---|
| 8 | CONTROLLERPORT=11071
|
|---|
| 9 | WORKERLISTENPORT=11072
|
|---|
| 10 |
|
|---|
| 11 | # start service in background
|
|---|
| 12 | ${AUTOTEST_PATH}/JobMarketServer --workeraddress 127.0.0.1:$WORKERLISTENPORT --workerport $WORKERPORT --controllerport $CONTROLLERPORT &
|
|---|
| 13 | server_pid=$!
|
|---|
| 14 | AT_CHECK([sleep 1], 0, [ignore], [ignore], [kill $server_pid])
|
|---|
| 15 |
|
|---|
| 16 | # add a worker to work on jobs
|
|---|
| 17 | ${AUTOTEST_PATH}/JobMarketPoolWorker -v 5 --signal 2 --server 127.0.0.1:${WORKERPORT} --hostname 127.0.0.1 --listen $WORKERLISTENPORT 2>workererr &
|
|---|
| 18 | worker_pid=$!
|
|---|
| 19 |
|
|---|
| 20 | # add one always failing job via JobAdder
|
|---|
| 21 | AT_CHECK([${AUTOTEST_PATH}/JobMarketController --server 127.0.0.1:$CONTROLLERPORT --command createjobs --executable "/bin/false" --jobcommand "Nothing"], 0, [stdout], [ignore], [kill $server_pid])
|
|---|
| 22 | AT_CHECK([fgrep "Sending 1 jobs ..." stdout], 0, [ignore], [ignore], [kill $server_pid])
|
|---|
| 23 | AT_CHECK([sleep 3], 0, [ignore], [ignore], [kill $server_pid $worker_pid])
|
|---|
| 24 |
|
|---|
| 25 | # send kill signal to worker such that it shuts down
|
|---|
| 26 | AT_CHECK([kill -INT $worker_pid], 0, [ignore], [ignore], [kill $server_pid $worker_pid])
|
|---|
| 27 |
|
|---|
| 28 | # check that it truely failed for four times
|
|---|
| 29 | AT_CHECK([fgrep -c "Job 1 failed on executing" workererr], 0, [stdout], [ignore], [kill $server_pid])
|
|---|
| 30 | AT_CHECK([grep "4" stdout], 0, [ignore], [ignore])
|
|---|
| 31 |
|
|---|
| 32 | # shutdown
|
|---|
| 33 | AT_CHECK([${AUTOTEST_PATH}/JobMarketController --server 127.0.0.1:$CONTROLLERPORT --command shutdown], 0, [ignore], [ignore], [kill $server_pid])
|
|---|
| 34 |
|
|---|
| 35 | # check that server is truely down
|
|---|
| 36 | AT_CHECK([sleep 1], 0, [ignore], [ignore], [kill $server_pid])
|
|---|
| 37 | AT_CHECK([kill $server_pid], 1, [ignore], [ignore])
|
|---|
| 38 |
|
|---|
| 39 | AT_CHECK([rm -f workererr], 0, [ignore], [ignore])
|
|---|
| 40 |
|
|---|
| 41 | AT_CLEANUP
|
|---|
