Created
November 12, 2019 08:35
-
-
Save worldwise001/c437c616f3b7c30f0b899d673d478047 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def __init__(self, config: Config, modules: List[str]): | |
kwargs = config.get('MYSQL_ENGINE_ARGS') | |
if not kwargs: | |
raise Exception('No MYSQL_ENGINE_ARGS set in setup_job_scheduler') | |
self.available_jobs: Dict[str, Any] = {} | |
# Import modules as necessary for the purpose of instantiating Jobs | |
for module_str in modules: | |
module = import_module(module_str) | |
if module is not None: | |
path = getattr(module, '__path__', None) | |
if path is not None: | |
for submodule_info in pkgutil.iter_modules(path): | |
submodule_str = f'{module.__name__}.{submodule_info.name}' | |
import_module(submodule_str) | |
# Instantiate all Job subclasses and add them to available_jobs | |
for job_class in Job.__subclasses__(): | |
should_be_loaded = False | |
for module_str in modules: | |
if job_class.__module__.startswith(module_str): | |
should_be_loaded = True | |
break | |
if not should_be_loaded: | |
continue | |
# TODO: type this | |
job = job_class(config) | |
job.origin = JobOrigin.CONFIG | |
self.available_jobs[job.get_id()] = job | |
# NB: If jobs are missed for some reason, still try to run within 120 minutes | |
self.scheduler = BackgroundScheduler(job_defaults={'misfire_grace_time': 120 * 60}) | |
self.scheduler.add_jobstore('sqlalchemy', engine=sqlalchemy.create_engine(**kwargs)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment