-
-
Save ritec/790bd4641513b13a4ef03231afa41866 to your computer and use it in GitHub Desktop.
Nested Batch workflow with Sidekiq Pro
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| class ImportJob | |
| include Sidekiq::Worker | |
| sidekiq_options :queue => :import, :retry => false, :backtrace => true | |
| def perform(project_id) | |
| # create master batch | |
| a = Sidekiq::Batch.new | |
| a.description = "Master Batch A" | |
| a.on(:success, "ImportJob#on_success", {"step" => "a"}) | |
| logger.info "Master Batch A starting #{a.bid}" | |
| a.jobs do | |
| b = Sidekiq::Batch.new | |
| b.description = "Batch B" | |
| b.on(:success, "ImportJob#on_success", 'step' => 'b') | |
| b.jobs do | |
| logger.info "+-- Batch B starting work #{b.bid}" | |
| SleepJob.perform_async | |
| SleepJob.perform_async | |
| end | |
| end | |
| end | |
| def on_success(status, options) | |
| case options['step'] | |
| when 'a' | |
| logger.info "+- Workflow finished" | |
| when 'b' | |
| logger.info "+-- Batch B finished. Starting next batches" | |
| Sidekiq::Batch.new(status.parent_bid).jobs do # add more jobs to A | |
| c = Sidekiq::Batch.new | |
| c.description = "Batch C" | |
| c.on(:success, "ImportJob#on_success", 'step' => 'c') | |
| c.jobs do | |
| logger.info "+---- Batch C starting work #{c.bid}" | |
| SleepJob.perform_async | |
| SleepJob.perform_async | |
| SleepJob.perform_async | |
| end | |
| d = Sidekiq::Batch.new | |
| d.description = "Batch D" | |
| d.on(:success, "ImportJob#on_success", 'step' => 'd') | |
| d.jobs do | |
| logger.info "+---- Batch D starting work #{d.bid}" | |
| SleepJob.perform_async | |
| end | |
| end | |
| when 'c' | |
| logger.info "+------ Batch C finished" | |
| when 'd' | |
| logger.info "+------ Batch D finished" | |
| else | |
| logger.debug status.inspect | |
| end | |
| end | |
| def on_complete(status, options) | |
| logger.debug "In on_complete -- #{status.inspect}" | |
| if status.failures != 0 | |
| logger.debug "Uh oh, batch has failures" | |
| logger.debug status.failures.inspect | |
| end | |
| end | |
| end |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2015-11-05T17:55:46.299Z 2933 TID-n7aa0 ImportJob JID-fdfcb44d09d632d642b2d0a7 INFO: start | |
| 2015-11-05T17:55:46.299Z 2933 TID-n7aa0 ImportJob JID-fdfcb44d09d632d642b2d0a7 INFO: Master Batch A starting 6APIUezUoOrXXg | |
| 2015-11-05T17:55:46.300Z 2933 TID-n7aa0 ImportJob JID-fdfcb44d09d632d642b2d0a7 INFO: +-- Batch B starting work rp_oXUPqEAINnQ | |
| 2015-11-05T17:55:46.302Z 2933 TID-n7aa0 ImportJob JID-fdfcb44d09d632d642b2d0a7 INFO: done: 0.003 sec | |
| 2015-11-05T17:55:46.373Z 2933 TID-ki37c SleepJob JID-b1b3082009c0b0d15598296c BID-rp_oXUPqEAINnQ INFO: start | |
| 2015-11-05T17:55:46.374Z 2933 TID-ouz9nctas SleepJob JID-c73283306bd69858c9aa155b BID-rp_oXUPqEAINnQ INFO: start | |
| 2015-11-05T17:55:58.375Z 2933 TID-ouz9nctas SleepJob JID-c73283306bd69858c9aa155b BID-rp_oXUPqEAINnQ INFO: done: 12.001 sec | |
| 2015-11-05T17:56:01.376Z 2933 TID-ki37c SleepJob JID-b1b3082009c0b0d15598296c BID-rp_oXUPqEAINnQ INFO: done: 15.003 sec | |
| 2015-11-05T17:56:01.377Z 2933 TID-ouz9nctas Sidekiq::Batch::Callback JID-7106ca6368906af7805ebb26 INFO: start | |
| 2015-11-05T17:56:01.380Z 2933 TID-ki37c Sidekiq::Batch::Callback JID-7341a5d54342e0d984f04013 INFO: start | |
| 2015-11-05T17:56:01.380Z 2933 TID-ouz9nctas Sidekiq::Batch::Callback JID-7106ca6368906af7805ebb26 INFO: done: 0.003 sec | |
| 2015-11-05T17:56:01.381Z 2933 TID-n7aa0 Sidekiq::Batch::Callback JID-e89943e0d9409f43f0f88017 INFO: start | |
| 2015-11-05T17:56:01.382Z 2933 TID-n7aa0 Sidekiq::Batch::Callback JID-e89943e0d9409f43f0f88017 INFO: +-- Batch B finished. Starting next batches | |
| 2015-11-05T17:56:01.382Z 2933 TID-n7aa0 Sidekiq::Batch::Callback JID-e89943e0d9409f43f0f88017 INFO: +---- Batch C starting work rHnEo88-BvskPA | |
| 2015-11-05T17:56:01.383Z 2933 TID-ki37c Sidekiq::Batch::Callback JID-7341a5d54342e0d984f04013 INFO: done: 0.004 sec | |
| 2015-11-05T17:56:01.385Z 2933 TID-n7aa0 Sidekiq::Batch::Callback JID-e89943e0d9409f43f0f88017 INFO: +---- Batch D starting work ug6SL-2ieGvhYg | |
| 2015-11-05T17:56:01.387Z 2933 TID-ki37c SleepJob JID-467e729534e9e79ad726aba1 BID-rHnEo88-BvskPA INFO: start | |
| 2015-11-05T17:56:01.460Z 2933 TID-ouz9nctas SleepJob JID-d9ea5900edd5b29d6eb89254 BID-rHnEo88-BvskPA INFO: start | |
| 2015-11-05T17:56:01.462Z 2933 TID-n7aa0 Sidekiq::Batch::Callback JID-e89943e0d9409f43f0f88017 INFO: done: 0.081 sec | |
| 2015-11-05T17:56:01.462Z 2933 TID-ll25g SleepJob JID-44d61668df63142a6bef79cb BID-rHnEo88-BvskPA INFO: start | |
| 2015-11-05T17:56:01.462Z 2933 TID-n8i6k SleepJob JID-be0f02c3db006eb079234143 BID-ug6SL-2ieGvhYg INFO: start | |
| 2015-11-05T17:56:10.463Z 2933 TID-ll25g SleepJob JID-44d61668df63142a6bef79cb BID-rHnEo88-BvskPA INFO: done: 9.002 sec | |
| 2015-11-05T17:56:11.462Z 2933 TID-ki37c SleepJob JID-467e729534e9e79ad726aba1 BID-rHnEo88-BvskPA INFO: done: 10.075 sec | |
| 2015-11-05T17:56:11.464Z 2933 TID-n8i6k SleepJob JID-be0f02c3db006eb079234143 BID-ug6SL-2ieGvhYg INFO: done: 10.002 sec | |
| 2015-11-05T17:56:11.465Z 2933 TID-ki37c Sidekiq::Batch::Callback JID-c201ef1efceb8bb5fee5793b INFO: start | |
| 2015-11-05T17:56:11.467Z 2933 TID-ki37c Sidekiq::Batch::Callback JID-c201ef1efceb8bb5fee5793b INFO: done: 0.002 sec | |
| 2015-11-05T17:56:11.467Z 2933 TID-n8i6k Sidekiq::Batch::Callback JID-0769b2064a6ba25935f03d4f INFO: start | |
| 2015-11-05T17:56:11.468Z 2933 TID-n8i6k Sidekiq::Batch::Callback JID-0769b2064a6ba25935f03d4f INFO: +------ Batch D finished | |
| 2015-11-05T17:56:11.468Z 2933 TID-n8i6k Sidekiq::Batch::Callback JID-0769b2064a6ba25935f03d4f INFO: done: 0.001 sec | |
| 2015-11-05T17:56:16.463Z 2933 TID-ouz9nctas SleepJob JID-d9ea5900edd5b29d6eb89254 BID-rHnEo88-BvskPA INFO: done: 15.003 sec | |
| 2015-11-05T17:56:16.464Z 2933 TID-n8i6k Sidekiq::Batch::Callback JID-e56ded55c0c7ef60aad63565 INFO: start | |
| 2015-11-05T17:56:16.466Z 2933 TID-n8i6k Sidekiq::Batch::Callback JID-e56ded55c0c7ef60aad63565 INFO: done: 0.002 sec | |
| 2015-11-05T17:56:16.466Z 2933 TID-ouz9nctas Sidekiq::Batch::Callback JID-55172c3293e1e38048eb50f5 INFO: start | |
| 2015-11-05T17:56:16.467Z 2933 TID-ouz9nctas Sidekiq::Batch::Callback JID-55172c3293e1e38048eb50f5 INFO: +------ Batch C finished | |
| 2015-11-05T17:56:16.468Z 2933 TID-ouz9nctas Sidekiq::Batch::Callback JID-55172c3293e1e38048eb50f5 INFO: done: 0.002 sec | |
| 2015-11-05T17:56:16.468Z 2933 TID-n8i6k Sidekiq::Batch::Callback JID-25c896ce33f6b9665a9c85e0 INFO: start | |
| 2015-11-05T17:56:16.469Z 2933 TID-n8i6k Sidekiq::Batch::Callback JID-25c896ce33f6b9665a9c85e0 INFO: +- Workflow finished | |
| 2015-11-05T17:56:16.469Z 2933 TID-n8i6k Sidekiq::Batch::Callback JID-25c896ce33f6b9665a9c85e0 INFO: done: 0.001 sec |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment