目標:
弄清楚st2rulesengine服務原理
1 總入口
st2/st2reactor/st2reactor/cmd/rulesengine.py
def main():
try:
_setup()
return _run_worker()
except SystemExit as exit_code:
sys.exit(exit_code)
except:
LOG.exception('(PID=%s) RulesEngine quit due to exception.', os.getpid())
return 1
finally:
_teardown()
1.1 進入_setup
def _setup():
common_setup(service='rulesengine', config=config, setup_db=True, register_mq_exchanges=True,
register_signal_handlers=True)
1.2 進入_run_worker
def _run_worker():
LOG.info('(PID=%s) RulesEngine started.', os.getpid())
timer = None
rules_engine_worker = worker.get_worker()
try:
timer_thread = None
if cfg.CONF.timer.enable:
timer = St2Timer(local_timezone=cfg.CONF.timer.local_timezone)
timer_thread = eventlet.spawn(_kickoff_timer, timer)
LOG.info(TIMER_ENABLED_LOG_LINE)
else:
LOG.info(TIMER_DISABLED_LOG_LINE)
rules_engine_worker.start()
if timer:
return timer_thread.wait() and rules_engine_worker.wait()
else:
return rules_engine_worker.wait()
except (KeyboardInterrupt, SystemExit):
LOG.info('(PID=%s) RulesEngine stopped.', os.getpid())
rules_engine_worker.shutdown()
except:
LOG.exception('(PID:%s) RulesEngine quit due to exception.', os.getpid())
return 1
finally:
if timer:
timer.cleanup()
return 0
分析:
1.2.1 分析
rules_engine_worker = worker.get_worker()
進入:
/opt/stackstorm/st2/lib/python2.7/site-packages/st2reactor/rules/worker.py(111)get_worker()
(Pdb) p rules_engine_worker
<st2reactor.rules.worker.TriggerInstanceDispatcher object at 0x2fe3a10>
(Pdb) p rules_engine_worker.__dict__
{'_consumer_thread': None, '_queue_consumer': <st2common.transport.consumers.StagedQueueConsumer object at 0x2fe3f90>, 'rules_engine': <st2reactor.rules.engine.RulesEngine object at 0x1642450>}
(Pdb) p type(rules_engine_worker)
<class 'st2reactor.rules.worker.TriggerInstanceDispatcher'>
分析:
這裏的rules_engine_worker實際上是消息處理類，是st2reactor.rules.worker.TriggerInstanceDispatcher對象。
TriggerInstanceDispatcher的基類是
st2/st2common/st2common/transport/consumers.py的MessageHandler類，這個類包含
_queue_consumer對象，這個對象包含處理消息的對象，即MessageHandler類的子類對象，每個繼承自
MessageHandler類的子類必須實現process(message)方法用於處理消息。
1.2.2 分析
timer = St2Timer(local_timezone=cfg.CONF.timer.local_timezone)
......
timer_thread = eventlet.spawn(_kickoff_timer, timer)
具體參見 2 的分析
1.2.3 分析
rules_engine_worker = worker.get_worker()
......
rules_engine_worker.start()
具體參見3的分析
2 分析St2Timer類的start方法
timer = St2Timer(local_timezone=cfg.CONF.timer.local_timezone)
......
timer_thread = eventlet.spawn(_kickoff_timer, timer)
其中:
def _kickoff_timer(timer):
timer.start()
2.1 分析
class St2Timer(object):
"""
A timer interface that uses APScheduler 3.0.
"""
def __init__(self, local_timezone=None):
self._timezone = local_timezone
self._scheduler = BlockingScheduler(timezone=self._timezone)
self._jobs = {}
self._trigger_types = TIMER_TRIGGER_TYPES.keys()
self._trigger_watcher = TriggerWatcher(create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix=self.__class__.__name__,
exclusive=True)
self._trigger_dispatcher = TriggerDispatcher(LOG)
def start(self):
self._register_timer_trigger_types()
self._trigger_watcher.start()
self._scheduler.start()
2.2)分析
self._register_timer_trigger_types()
進入:
def _register_timer_trigger_types(self):
return trigger_services.add_trigger_models(TIMER_TRIGGER_TYPES.values())
分析:
_register_timer_trigger_types(self)調用add_trigger_models(trigger_types):
2.2.1) 分析
進入: st2/st2common/st2common/services/triggers.py
def add_trigger_models(trigger_types):
"""
Register trigger types.
:param trigger_types: A list of triggers to register.
:type trigger_types: ``list`` of ``dict``
:rtype: ``list`` of ``tuple`` (trigger_type, trigger)
"""
[r for r in (_validate_trigger_type(trigger_type)
for trigger_type in trigger_types) if r is not None]
result = []
for trigger_type in trigger_types:
item = _add_trigger_models(trigger_type=trigger_type)
if item:
result.append(item)
return result
分析:
add_trigger_models(trigger_types):
1 根據輸入參數，形如:
(Pdb) p trigger_types
[{'payload_schema': {'type': 'object', 'properties': {'executed_at': {'default': '2014-07-30 05:04:24.578325', 'type': 'string', 'format': 'date-time'}, 'schedule': {'default': {'units': 'seconds', 'delta': 30}, 'type': 'object'}}}, 'description': 'Triggers on specified intervals. e.g. every 30s, 1week etc.', 'parameters_schema': {'additionalProperties': False, 'type': 'object', 'properties': {'timezone': {'type': 'string'}, 'unit': {'enum': ['weeks', 'days', 'hours', 'minutes', 'seconds'], 'required': True}, 'delta': {'required': True, 'type': 'integer'}}}, 'name': 'st2.IntervalTimer', 'pack': 'core'}, {'payload_schema': {'type': 'object', 'properties': {'executed_at': {'default': '2014-07-30 05:04:24.578325', 'type': 'string', 'format': 'date-time'}, 'schedule': {'default': {'units': 'seconds', 'delta': 30}, 'type': 'object'}}}, 'description': 'Triggers exactly once when the current time matches the specified time. e.g. timezone:UTC date:2014-12-31 23:59:59.', 'parameters_schema': {'additionalProperties': False, 'type': 'object', 'properties': {'date': {'required': True, 'type': 'string', 'format': 'date-time'}, 'timezone': {'type': 'string'}}}, 'name': 'st2.DateTimer', 'pack': 'core'}, {'payload_schema': {'type': 'object', 'properties': {'executed_at': {'default': '2014-07-30 05:04:24.578325', 'type': 'string', 'format': 'date-time'}, 'schedule': {'default': {'units': 'seconds', 'delta': 30}, 'type': 'object'}}}, 'description': 'Triggers whenever current time matches the specified time constaints like a UNIX cron scheduler.', 'parameters_schema': {'additionalProperties': False, 'type': 'object', 'properties': {'week': {'minimum': 1, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 53}, 'end_date': {'type': 'string', 'format': 'date-time'}, 'month': {'minimum': 1, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 12}, 'second': {'minimum': 0, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 59}, 'year': {'anyOf': [{'type': 'string'}, {'type': 'integer'}]}, 'timezone': {'type': 'string'}, 'day': {'minimum': 1, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 31}, 'minute': {'minimum': 0, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 59}, 'hour': {'minimum': 0, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 23}, 'day_of_week': {'minimum': 0, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 6}, 'start_date': {'type': 'string', 'format': 'date-time'}}}, 'name': 'st2.CronTimer', 'pack': 'core'}]
2 遍歷trigger_types列表，，每個trigger_type，調用_add_trigger_models(trigger_type)
2.2.2) 分析_add_trigger_models方法
def _add_trigger_models(trigger_type):
pack = trigger_type['pack']
description = trigger_type['description'] if 'description' in trigger_type else ''
payload_schema = trigger_type['payload_schema'] if 'payload_schema' in trigger_type else {}
parameters_schema = trigger_type['parameters_schema'] \
if 'parameters_schema' in trigger_type else {}
tags = trigger_type.get('tags', [])
trigger_type = _create_trigger_type(
pack=pack,
name=trigger_type['name'],
description=description,
payload_schema=payload_schema,
parameters_schema=parameters_schema,
tags=tags
)
trigger = _create_trigger(trigger_type=trigger_type)
return (trigger_type, trigger)
分析:
_add_trigger_models(trigger_type):
1 根據輸入參數，形如:
(Pdb) p trigger_type
{'payload_schema': {'type': 'object', 'properties': {'executed_at': {'default': '2014-07-30 05:04:24.578325', 'type': 'string', 'format': 'date-time'}, 'schedule': {'default': {'units': 'seconds', 'delta': 30}, 'type': 'object'}}}, 'description': 'Triggers on specified intervals. e.g. every 30s, 1week etc.', 'parameters_schema': {'additionalProperties': False, 'type': 'object', 'properties': {'timezone': {'type': 'string'}, 'unit': {'enum': ['weeks', 'days', 'hours', 'minutes', 'seconds'], 'required': True}, 'delta': {'required': True, 'type': 'integer'}}}, 'name': 'st2.IntervalTimer', 'pack': 'core'}
2 調用_create_trigger_type(pack, name, description=None, payload_schema=None, parameters_schema=None, tags=None)
2.2.3) 分析_create_trigger_type方法
def _create_trigger_type(pack, name, description=None, payload_schema=None,
parameters_schema=None, tags=None):
trigger_type = {
'name': name,
'pack': pack,
'description': description,
'payload_schema': payload_schema,
'parameters_schema': parameters_schema,
'tags': tags
}
return create_or_update_trigger_type_db(trigger_type=trigger_type)
分析:
_create_trigger_type(pack, name, description=None, payload_schema=None, parameters_schema=None, tags=None):
1 根據輸入參數，形如:
(Pdb) p pack
'core'
(Pdb) p name
'st2.IntervalTimer'
(Pdb) p description
'Triggers on specified intervals. e.g. every 30s, 1week etc.'
(Pdb) p payload_schema
{'type': 'object', 'properties': {'executed_at': {'default': '2014-07-30 05:04:24.578325', 'type': 'string', 'format': 'date-time'}, 'schedule': {'default': {'units': 'seconds', 'delta': 30}, 'type': 'object'}}}
(Pdb) p parameters_schema
{'additionalProperties': False, 'type': 'object', 'properties': {'timezone': {'type': 'string'}, 'unit': {'enum': ['weeks', 'days', 'hours', 'minutes', 'seconds'], 'required': True}, 'delta': {'required': True, 'type': 'integer'}}}
(Pdb) p tags
[]
2 創建並返回TriggerTypeDB對象並寫入數據庫，
返回的結果樣例如下:
<TriggerTypeDB: TriggerTypeDB(description="Triggers on specified intervals. e.g. every 30s, 1week etc.", id=5e6603b9a7918a0001b8a3b3, name="st2.IntervalTimer", pack="core", parameters_schema={'additionalProperties': False, 'type': 'object', 'properties': {'timezone': {'type': 'string'}, 'unit': {'enum': ['weeks', 'days', 'hours', 'minutes', 'seconds'], 'required': True}, 'delta': {'required': True, 'type': 'integer'}}}, payload_schema={'type': 'object', 'properties': {'executed_at': {'default': '2014-07-30 05:04:24.578325', 'type': 'string', 'format': 'date-time'}, 'schedule': {'default': {'units': 'seconds', 'delta': 30}, 'type': 'object'}}}, ref="core.st2.IntervalTimer", tags=[], uid="trigger_type:core:st2.IntervalTimer")>
2.2.4)分析_create_trigger方法
def _create_trigger(trigger_type):
"""
:param trigger_type: TriggerType db object.
:type trigger_type: :class:`TriggerTypeDB`
"""
if hasattr(trigger_type, 'parameters_schema') and not trigger_type['parameters_schema']:
trigger_dict = {
'name': trigger_type.name,
'pack': trigger_type.pack,
'type': trigger_type.get_reference().ref
}
try:
return create_or_update_trigger_db(trigger=trigger_dict)
except:
LOG.exception('Validation failed for Trigger=%s.', trigger_dict)
raise TriggerTypeRegistrationException(
'Unable to create Trigger for TriggerType=%s.' % trigger_type.name)
else:
LOG.debug('Won\'t create Trigger object as TriggerType %s expects ' +
'parameters.', trigger_type)
return None
2.3)分析
self._trigger_watcher.start()
進入st2/st2common/st2common/services/triggerwatcher.py
class TriggerWatcher(ConsumerMixin):
def __init__(self, create_handler, update_handler, delete_handler,
trigger_types=None, queue_suffix=None, exclusive=False):
# TODO: Handle trigger type filtering using routing key
self._create_handler = create_handler
self._update_handler = update_handler
self._delete_handler = delete_handler
self._trigger_types = trigger_types
self._trigger_watch_q = self._get_queue(queue_suffix, exclusive=exclusive)
self.connection = None
self._load_thread = None
self._updates_thread = None
self._handlers = {
publishers.CREATE_RK: create_handler,
publishers.UPDATE_RK: update_handler,
publishers.DELETE_RK: delete_handler
}
def start(self):
try:
self.connection = Connection(transport_utils.get_messaging_urls())
self._updates_thread = eventlet.spawn(self.run)
self._load_thread = eventlet.spawn(self._load_triggers_from_db)
except:
LOG.exception('Failed to start watcher.')
self.connection.release()
def get_consumers(self, Consumer, channel):
return [Consumer(queues=[self._trigger_watch_q],
accept=['pickle'],
callbacks=[self.process_task])]
分析:
2.3.0)
self._trigger_watch_q = self._get_queue(queue_suffix, exclusive=exclusive)
分析:
(Pdb) p self._trigger_watch_q
<unbound Queue st2.trigger.watch.St2Timer-3ee1dd8cd0 -> <unbound Exchange st2.trigger(topic)> -> #>
(Pdb) p self._trigger_watch_q.__dict__
{'no_declare': None, 'exclusive': True, 'on_declared': None, 'message_ttl': None, 'exchange': <unbound Exchange st2.trigger(topic)>, 'auto_delete': True, 'expires': None, 'routing_key': '#', 'name': 'st2.trigger.watch.St2Timer-3ee1dd8cd0', 'alias': None, 'queue_arguments': None, 'max_length_bytes': None, 'binding_arguments': None, 'max_length': None, 'bindings': set([]), 'consumer_arguments': None, 'max_priority': None}
(Pdb) p type(self._trigger_watch_q)
<class 'kombu.entity.Queue'>
2.3.1)
TriggerWatcher(ConsumerMixin).start()方法
1 初始化消息隊列的連接
2 開啓協程去執行run方法，具體是:
調用/usr/lib/python2.7/site-packages/kombu/mixins.py文件的ConsumerMixin類的run方法來消費消息
3 開啓協程去執行_load_triggers_from_db方法，具體是:
_load_triggers_from_db(self):
遍歷每個trigger_type，查詢該trigger_type對應的trigger信息，形如:
<TriggerDB: TriggerDB(description=None, id=5e78a2f0618fd301a43e2284, name="8c13d9a3-12a3-4cff-88d3-66c804a47590", pack="core", parameters={u'unit': u'minutes', u'delta': 5}, ref="core.8c13d9a3-12a3-4cff-88d3-66c804a47590", ref_count=1, type="core.st2.IntervalTimer", uid="trigger:core:8c13d9a3-12a3-4cff-88d3-66c804a47590:5e0acee19fa878c4fa07d596f7a893d6")>
調用create_handler(trigger)方法，實際就是調用St2Timer._handle_create_trigger(trigger)
2.3.2) 分析
self._load_thread = eventlet.spawn(self._load_triggers_from_db)
進入:
def _load_triggers_from_db(self):
for trigger_type in self._trigger_types:
for trigger in Trigger.query(type=trigger_type):
LOG.debug('Found existing trigger: %s in db.' % trigger)
self._handlers[publishers.CREATE_RK](trigger)
分析:
_load_triggers_from_db(self):
遍歷每個trigger_type，查詢該trigger_type對應的trigger信息，形如:
<TriggerDB: TriggerDB(description=None, id=5e78a2f0618fd301a43e2284, name="8c13d9a3-12a3-4cff-88d3-66c804a47590", pack="core", parameters={u'unit': u'minutes', u'delta': 5}, ref="core.8c13d9a3-12a3-4cff-88d3-66c804a47590", ref_count=1, type="core.st2.IntervalTimer", uid="trigger:core:8c13d9a3-12a3-4cff-88d3-66c804a47590:5e0acee19fa878c4fa07d596f7a893d6")>
調用create_handler(trigger)方法，實際就是調用St2Timer._handle_create_trigger(trigger)
2.3.3) 分析St2Timer._handle_create_trigger(trigger)
class St2Timer(object):
def _handle_create_trigger(self, trigger):
LOG.debug('Calling "add_trigger" method (trigger.type=%s)' % (trigger.type))
trigger = self._sanitize_trigger(trigger=trigger)
self.add_trigger(trigger=trigger)
分析:
_handle_create_trigger(self, trigger):
1 根據輸入參數，樣例如下:
<TriggerDB: TriggerDB(description=None, id=5e78a2f0618fd301a43e2284, name="8c13d9a3-12a3-4cff-88d3-66c804a47590", pack="core", parameters={u'unit': u'minutes', u'delta': 5}, ref="core.8c13d9a3-12a3-4cff-88d3-66c804a47590", ref_count=1, type="core.st2.IntervalTimer", uid="trigger:core:8c13d9a3-12a3-4cff-88d3-66c804a47590:5e0acee19fa878c4fa07d596f7a893d6")>
2 對trigger對象，調用add_trigger方法,
2.3.4) 分析add_trigger方法
class St2Timer(object):
def add_trigger(self, trigger):
self._add_job_to_scheduler(trigger)
def _add_job_to_scheduler(self, trigger):
trigger_type_ref = trigger['type']
trigger_type = TIMER_TRIGGER_TYPES[trigger_type_ref]
try:
util_schema.validate(instance=trigger['parameters'],
schema=trigger_type['parameters_schema'],
cls=util_schema.CustomValidator,
use_default=True,
allow_default_none=True)
except jsonschema.ValidationError as e:
LOG.error('Exception scheduling timer: %s, %s',
trigger['parameters'], e, exc_info=True)
raise # Or should we just return?
time_spec = trigger['parameters']
time_zone = aps_utils.astimezone(trigger['parameters'].get('timezone'))
time_type = None
if trigger_type['name'] == 'st2.IntervalTimer':
unit = time_spec.get('unit', None)
value = time_spec.get('delta', None)
time_type = IntervalTrigger(**{unit: value, 'timezone': time_zone})
elif trigger_type['name'] == 'st2.DateTimer':
# Raises an exception if date string isn't a valid one.
dat = date_parser.parse(time_spec.get('date', None))
time_type = DateTrigger(dat, timezone=time_zone)
elif trigger_type['name'] == 'st2.CronTimer':
cron = time_spec.copy()
cron['timezone'] = time_zone
time_type = CronTrigger(**cron)
utc_now = date_utils.get_datetime_utc_now()
if hasattr(time_type, 'run_date') and utc_now > time_type.run_date:
LOG.warning('Not scheduling expired timer: %s : %s',
trigger['parameters'], time_type.run_date)
else:
self._add_job(trigger, time_type)
return time_type
分析:
add_trigger方法調用_add_job_to_scheduler方法，
_add_job_to_scheduler方法: 獲取trigger類型(例如: 'core.st2.IntervalTimer')，參數(例如unit: minutes, delta: 5), 實例化對應apscheduler中trigger對象
(例如: IntervalTrigger)， 向apscheduler的BlockingScheduler對象中添加job,job執行_emit_trigger_instance方法並攜帶參數trigger。
具體是調用add_trigger(self, trigger):
1 根據輸入參數，形如:
(Pdb) p trigger
{'uid': u'trigger:core:8c13d9a3-12a3-4cff-88d3-66c804a47590:5e0acee19fa878c4fa07d596f7a893d6', 'parameters': {u'unit': u'minutes', u'delta': 5}, 'ref': u'core.8c13d9a3-12a3-4cff-88d3-66c804a47590', 'name': u'8c13d9a3-12a3-4cff-88d3-66c804a47590', 'type': u'core.st2.IntervalTimer', 'id': '5e78a2f0618fd301a43e2284', 'pack': u'core'}
2 調用_add_job_to_scheduler(self, trigger):
s1 根據輸入參數，形如:
(Pdb) p trigger
{'uid': u'trigger:core:8c13d9a3-12a3-4cff-88d3-66c804a47590:5e0acee19fa878c4fa07d596f7a893d6', 'parameters': {u'unit': u'minutes', u'delta': 5}, 'ref': u'core.8c13d9a3-12a3-4cff-88d3-66c804a47590', 'name': u'8c13d9a3-12a3-4cff-88d3-66c804a47590', 'type': u'core.st2.IntervalTimer', 'id': '5e78a2f0618fd301a43e2284', 'pack': u'core'}
s2 獲取trigger類型(例如: 'core.st2.IntervalTimer')，參數(例如unit: minutes, delta: 5), 實例化對應apscheduler中trigger對象
(例如: IntervalTrigger, DateTrigger, CronTrigger)
s3 調用 _add_job(self, trigger, time_type, replace=True)方法來添加定時任務
2.3.5)分析
class St2Timer(object):
def _add_job(self, trigger, time_type, replace=True):
try:
job = self._scheduler.add_job(self._emit_trigger_instance,
trigger=time_type,
args=[trigger],
replace_existing=replace)
LOG.info('Job %s scheduled.', job.id)
self._jobs[trigger['id']] = job.id
except Exception as e:
LOG.error('Exception scheduling timer: %s, %s',
trigger['parameters'], e, exc_info=True)
分析:
其中 self._scheduler是BlockingScheduler(timezone=self._timezone)
_add_job(self, trigger, time_type, replace=True):
1 根據輸入參數，形如:
(Pdb) p trigger
{'uid': u'trigger:core:8c13d9a3-12a3-4cff-88d3-66c804a47590:5e0acee19fa878c4fa07d596f7a893d6', 'parameters': {u'unit': u'minutes', u'delta': 5}, 'ref': u'core.8c13d9a3-12a3-4cff-88d3-66c804a47590', 'name': u'8c13d9a3-12a3-4cff-88d3-66c804a47590', 'type': u'core.st2.IntervalTimer', 'id': '5e78a2f0618fd301a43e2284', 'pack': u'core'}
(Pdb) p time_type
<IntervalTrigger (interval=datetime.timedelta(0, 300), start_date='2020-03-25 17:30:28 CST', timezone='Asia/Shanghai')>
(Pdb) p replace
True
2 向BlockingScheduler對象中添加job,jon執行_emit_trigger_instance方法並攜帶參數trigger。
3 _emit_trigger_instance方法發送消息到隊列上
2.3.6) 分析_emit_trigger_instance
def _emit_trigger_instance(self, trigger):
utc_now = date_utils.get_datetime_utc_now()
# debug logging is reasonable for this one. A high resolution timer will end up
# trashing standard logs.
LOG.debug('Timer fired at: %s. Trigger: %s', str(utc_now), trigger)
payload = {
'executed_at': str(utc_now),
'schedule': trigger['parameters'].get('time')
}
trace_context = TraceContext(trace_tag='%s-%s' % (self._get_trigger_type_name(trigger),
trigger.get('name', uuid.uuid4().hex)))
self._trigger_dispatcher.dispatch(trigger, payload, trace_context=trace_context)
分析:
_emit_trigger_instance(self, trigger):
1 根據輸入參數，形如:
(Pdb) p trigger
{'uid': u'trigger:core:8c13d9a3-12a3-4cff-88d3-66c804a47590:5e0acee19fa878c4fa07d596f7a893d6', 'parameters': {u'unit': u'minutes', u'delta': 5}, 'ref': u'core.8c13d9a3-12a3-4cff-88d3-66c804a47590', 'name': u'8c13d9a3-12a3-4cff-88d3-66c804a47590', 'type': u'core.st2.IntervalTimer', 'id': '5e78a2f0618fd301a43e2284', 'pack': u'core'}
2 生成payload字典，樣例如下:
{'executed_at': '2020-03-25 10:34:14.085891+00:00', 'schedule': None}
3 構建TraceContext對象(其trace-tag由trigger類型和trigger name拼接而成)，樣例如下:
{'id_': None, 'trace_tag': u'st2.IntervalTimer-8c13d9a3-12a3-4cff-88d3-66c804a47590'}
4 調用TriggerDispatcher類的dispatch(trigger, payload, trace_context=trace_context)方法，具體是:
2.3.7)
class TriggerDispatcher(object):
"""
This trigger dispatcher dispatches trigger instances to a message queue (RabbitMQ).
"""
def __init__(self, logger=LOG):
self._publisher = TriggerInstancePublisher(urls=transport_utils.get_messaging_urls())
self._logger = logger
def dispatch(self, trigger, payload=None, trace_context=None):
"""
Method which dispatches the trigger.
:param trigger: Full name / reference of the trigger.
:type trigger: ``str`` or ``object``
:param payload: Trigger payload.
:type payload: ``dict``
:param trace_context: Trace context to associate with Trigger.
:type trace_context: ``TraceContext``
"""
assert isinstance(payload, (type(None), dict))
assert isinstance(trace_context, (type(None), TraceContext))
payload = {
'trigger': trigger,
'payload': payload,
TRACE_CONTEXT: trace_context
}
routing_key = 'trigger_instance'
self._logger.debug('Dispatching trigger (trigger=%s,payload=%s)', trigger, payload)
self._publisher.publish_trigger(payload=payload, routing_key=routing_key)
分析:
TriggerDispatcher.dispatch(self, trigger, payload=None, trace_context=None):
1 根據輸入參數，形如:
(Pdb) p trigger
{'uid': u'trigger:core:8c13d9a3-12a3-4cff-88d3-66c804a47590:5e0acee19fa878c4fa07d596f7a893d6', 'parameters': {u'unit': u'minutes', u'delta': 5}, 'ref': u'core.8c13d9a3-12a3-4cff-88d3-66c804a47590', 'name': u'8c13d9a3-12a3-4cff-88d3-66c804a47590', 'type': u'core.st2.IntervalTimer', 'id': '5e78a2f0618fd301a43e2284', 'pack': u'core'}
(Pdb) p payload
{'executed_at': '2020-03-25 10:55:01.848503+00:00', 'schedule': None}
(Pdb) p trace_context
<st2common.models.api.trace.TraceContext object at 0x3be4f90>
(Pdb) p trace_context.__dict__
{'id_': None, 'trace_tag': u'st2.IntervalTimer-8c13d9a3-12a3-4cff-88d3-66c804a47590'}
2 構建payload,形如:
{'trace_context': <st2common.models.api.trace.TraceContext object at 0x3be4f90>, 'trigger': {'uid': u'trigger:core:8c13d9a3-12a3-4cff-88d3-66c804a47590:5e0acee19fa878c4fa07d596f7a893d6', 'parameters': {u'unit': u'minutes', u'delta': 5}, 'ref': u'core.8c13d9a3-12a3-4cff-88d3-66c804a47590', 'name': u'8c13d9a3-12a3-4cff-88d3-66c804a47590', 'type': u'core.st2.IntervalTimer', 'id': '5e78a2f0618fd301a43e2284', 'pack': u'core'}, 'payload': {'executed_at': '2020-03-25 10:55:01.848503+00:00', 'schedule': None}}
3 設置routing_key爲'trigger_instance'
4 調用TriggerInstancePublisher類的publish_trigger(payload=payload, routing_key=routing_key)方法，
2.3.8)分析TriggerInstancePublisher類的publish_trigger方法
class TriggerInstancePublisher(object):
def __init__(self, urls):
self._publisher = publishers.PoolPublisher(urls=urls)
def publish_trigger(self, payload=None, routing_key=None):
# TODO: We should use trigger reference as a routing key
self._publisher.publish(payload, TRIGGER_INSTANCE_XCHG, routing_key)
分析:
TriggerInstancePublisher.publish_trigger(self, payload=None, routing_key=None):
1 根據輸入參數，形如:
(Pdb) p payload
{'trace_context': <st2common.models.api.trace.TraceContext object at 0x3be4f90>, 'trigger': {'uid': u'trigger:core:8c13d9a3-12a3-4cff-88d3-66c804a47590:5e0acee19fa878c4fa07d596f7a893d6', 'parameters': {u'unit': u'minutes', u'delta': 5}, 'ref': u'core.8c13d9a3-12a3-4cff-88d3-66c804a47590', 'name': u'8c13d9a3-12a3-4cff-88d3-66c804a47590', 'type': u'core.st2.IntervalTimer', 'id': '5e78a2f0618fd301a43e2284', 'pack': u'core'}, 'payload': {'executed_at': '2020-03-25 10:55:01.848503+00:00', 'schedule': None}}
(Pdb) p routing_key
'trigger_instance'
2 調用PoolPublisher類的publish(payload, TRIGGER_INSTANCE_XCHG, routing_key)方法發送消息，具體是:
exchange類型是topic,名稱是: st2.trigger_instances_dispatch
routing_key是'trigger_instance'
2.4) 回到
class St2Timer(object):
def __init__(self, local_timezone=None):
self._timezone = local_timezone
self._scheduler = BlockingScheduler(timezone=self._timezone)
self._jobs = {}
self._trigger_types = TIMER_TRIGGER_TYPES.keys()
self._trigger_watcher = TriggerWatcher(create_handler=self._handle_create_trigger,
update_handler=self._handle_update_trigger,
delete_handler=self._handle_delete_trigger,
trigger_types=self._trigger_types,
queue_suffix=self.__class__.__name__,
exclusive=True)
self._trigger_dispatcher = TriggerDispatcher(LOG)
def start(self):
self._register_timer_trigger_types()
self._trigger_watcher.start()
self._scheduler.start()
================================
================================
總結:
St2Timer.start()處理流程如下:
1 註冊trigger類型列表，具體是:
調用add_trigger_models(trigger_types):
1 根據輸入參數，形如:
(Pdb) p trigger_types
[{'payload_schema': {'type': 'object', 'properties': {'executed_at': {'default': '2014-07-30 05:04:24.578325', 'type': 'string', 'format': 'date-time'}, 'schedule': {'default': {'units': 'seconds', 'delta': 30}, 'type': 'object'}}}, 'description': 'Triggers on specified intervals. e.g. every 30s, 1week etc.', 'parameters_schema': {'additionalProperties': False, 'type': 'object', 'properties': {'timezone': {'type': 'string'}, 'unit': {'enum': ['weeks', 'days', 'hours', 'minutes', 'seconds'], 'required': True}, 'delta': {'required': True, 'type': 'integer'}}}, 'name': 'st2.IntervalTimer', 'pack': 'core'}, {'payload_schema': {'type': 'object', 'properties': {'executed_at': {'default': '2014-07-30 05:04:24.578325', 'type': 'string', 'format': 'date-time'}, 'schedule': {'default': {'units': 'seconds', 'delta': 30}, 'type': 'object'}}}, 'description': 'Triggers exactly once when the current time matches the specified time. e.g. timezone:UTC date:2014-12-31 23:59:59.', 'parameters_schema': {'additionalProperties': False, 'type': 'object', 'properties': {'date': {'required': True, 'type': 'string', 'format': 'date-time'}, 'timezone': {'type': 'string'}}}, 'name': 'st2.DateTimer', 'pack': 'core'}, {'payload_schema': {'type': 'object', 'properties': {'executed_at': {'default': '2014-07-30 05:04:24.578325', 'type': 'string', 'format': 'date-time'}, 'schedule': {'default': {'units': 'seconds', 'delta': 30}, 'type': 'object'}}}, 'description': 'Triggers whenever current time matches the specified time constaints like a UNIX cron scheduler.', 'parameters_schema': {'additionalProperties': False, 'type': 'object', 'properties': {'week': {'minimum': 1, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 53}, 'end_date': {'type': 'string', 'format': 'date-time'}, 'month': {'minimum': 1, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 12}, 'second': {'minimum': 0, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 59}, 'year': {'anyOf': [{'type': 'string'}, {'type': 'integer'}]}, 'timezone': {'type': 'string'}, 'day': {'minimum': 1, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 31}, 'minute': {'minimum': 0, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 59}, 'hour': {'minimum': 0, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 23}, 'day_of_week': {'minimum': 0, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 6}, 'start_date': {'type': 'string', 'format': 'date-time'}}}, 'name': 'st2.CronTimer', 'pack': 'core'}]
2 遍歷trigger_types列表，每個trigger_type，調用
_add_trigger_models(trigger_type):
s1 根據輸入參數，形如:
(Pdb) p trigger_type
{'payload_schema': {'type': 'object', 'properties': {'executed_at': {'default': '2014-07-30 05:04:24.578325', 'type': 'string', 'format': 'date-time'}, 'schedule': {'default': {'units': 'seconds', 'delta': 30}, 'type': 'object'}}}, 'description': 'Triggers on specified intervals. e.g. every 30s, 1week etc.', 'parameters_schema': {'additionalProperties': False, 'type': 'object', 'properties': {'timezone': {'type': 'string'}, 'unit': {'enum': ['weeks', 'days', 'hours', 'minutes', 'seconds'], 'required': True}, 'delta': {'required': True, 'type': 'integer'}}}, 'name': 'st2.IntervalTimer', 'pack': 'core'}
s2 調用_create_trigger_type(pack, name, description=None, payload_schema=None, parameters_schema=None, tags=None):
創建並返回TriggerTypeDB對象並寫入數據庫，
返回的結果樣例如下:
<TriggerTypeDB: TriggerTypeDB(description="Triggers on specified intervals. e.g. every 30s, 1week etc.", id=5e6603b9a7918a0001b8a3b3, name="st2.IntervalTimer", pack="core", parameters_schema={'additionalProperties': False, 'type': 'object', 'properties': {'timezone': {'type': 'string'}, 'unit': {'enum': ['weeks', 'days', 'hours', 'minutes', 'seconds'], 'required': True}, 'delta': {'required': True, 'type': 'integer'}}}, payload_schema={'type': 'object', 'properties': {'executed_at': {'default': '2014-07-30 05:04:24.578325', 'type': 'string', 'format': 'date-time'}, 'schedule': {'default': {'units': 'seconds', 'delta': 30}, 'type': 'object'}}}, ref="core.st2.IntervalTimer", tags=[], uid="trigger_type:core:st2.IntervalTimer")>
3 返回trigger_type列表，樣例如下:
[(<TriggerTypeDB: TriggerTypeDB(description="Triggers on specified intervals. e.g. every 30s, 1week etc.", id=5e6603b9a7918a0001b8a3b3, name="st2.IntervalTimer", pack="core", parameters_schema={'additionalProperties': False, 'type': 'object', 'properties': {'timezone': {'type': 'string'}, 'unit': {'enum': ['weeks', 'days', 'hours', 'minutes', 'seconds'], 'required': True}, 'delta': {'required': True, 'type': 'integer'}}}, payload_schema={'type': 'object', 'properties': {'executed_at': {'default': '2014-07-30 05:04:24.578325', 'type': 'string', 'format': 'date-time'}, 'schedule': {'default': {'units': 'seconds', 'delta': 30}, 'type': 'object'}}}, ref="core.st2.IntervalTimer", tags=[], uid="trigger_type:core:st2.IntervalTimer")>, None), (<TriggerTypeDB: TriggerTypeDB(description="Triggers exactly once when the current time matches the specified time. e.g. timezone:UTC date:2014-12-31 23:59:59.", id=5e6603b9a7918a0001b8a3b4, name="st2.DateTimer", pack="core", parameters_schema={'additionalProperties': False, 'type': 'object', 'properties': {'date': {'required': True, 'type': 'string', 'format': 'date-time'}, 'timezone': {'type': 'string'}}}, payload_schema={'type': 'object', 'properties': {'executed_at': {'default': '2014-07-30 05:04:24.578325', 'type': 'string', 'format': 'date-time'}, 'schedule': {'default': {'units': 'seconds', 'delta': 30}, 'type': 'object'}}}, ref="core.st2.DateTimer", tags=[], uid="trigger_type:core:st2.DateTimer")>, None), (<TriggerTypeDB: TriggerTypeDB(description="Triggers whenever current time matches the specified time constaints like a UNIX cron scheduler.", id=5e6603b9a7918a0001b8a3b5, name="st2.CronTimer", pack="core", parameters_schema={'additionalProperties': False, 'type': 'object', 'properties': {'week': {'minimum': 1, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 53}, 'hour': {'minimum': 0, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 23}, 'end_date': {'type': 'string', 'format': 'date-time'}, 'day_of_week': {'minimum': 0, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 6}, 'month': {'minimum': 1, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 12}, 'second': {'minimum': 0, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 59}, 'minute': {'minimum': 0, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 59}, 'year': {'anyOf': [{'type': 'string'}, {'type': 'integer'}]}, 'timezone': {'type': 'string'}, 'start_date': {'type': 'string', 'format': 'date-time'}, 'day': {'minimum': 1, 'anyOf': [{'type': 'string'}, {'type': 'integer'}], 'maximum': 31}}}, payload_schema={'type': 'object', 'properties': {'executed_at': {'default': '2014-07-30 05:04:24.578325', 'type': 'string', 'format': 'date-time'}, 'schedule': {'default': {'units': 'seconds', 'delta': 30}, 'type': 'object'}}}, ref="core.st2.CronTimer", tags=[], uid="trigger_type:core:st2.CronTimer")>, None)]
2 調用TriggerWatcher(ConsumerMixin).start()方法
1 初始化消息隊列的連接
2 開啓協程去執行run方法，具體是:
調用/usr/lib/python2.7/site-packages/kombu/mixins.py文件的ConsumerMixin類的run方法來消費消息
3 開啓協程去執行_load_triggers_from_db方法，具體是:
_load_triggers_from_db(self):
遍歷每個trigger_type，查詢該trigger_type對應的trigger信息，形如:
<TriggerDB: TriggerDB(description=None, id=5e78a2f0618fd301a43e2284, name="8c13d9a3-12a3-4cff-88d3-66c804a47590", pack="core", parameters={u'unit': u'minutes', u'delta': 5}, ref="core.8c13d9a3-12a3-4cff-88d3-66c804a47590", ref_count=1, type="core.st2.IntervalTimer", uid="trigger:core:8c13d9a3-12a3-4cff-88d3-66c804a47590:5e0acee19fa878c4fa07d596f7a893d6")>
調用create_handler(trigger)方法，實際就是調用St2Timer._handle_create_trigger(trigger)
具體是:
_handle_create_trigger(self, trigger):
1 根據輸入參數，樣例如下:
<TriggerDB: TriggerDB(description=None, id=5e78a2f0618fd301a43e2284, name="8c13d9a3-12a3-4cff-88d3-66c804a47590", pack="core", parameters={u'unit': u'minutes', u'delta': 5}, ref="core.8c13d9a3-12a3-4cff-88d3-66c804a47590", ref_count=1, type="core.st2.IntervalTimer", uid="trigger:core:8c13d9a3-12a3-4cff-88d3-66c804a47590:5e0acee19fa878c4fa07d596f7a893d6")>
2 對trigger對象，調用add_trigger方法,add_trigger方法調用_add_job_to_scheduler方法，
_add_job_to_scheduler方法: 獲取trigger類型(例如: 'core.st2.IntervalTimer')，參數(例如unit: minutes, delta: 5), 實例化對應apscheduler中trigger對象
(例如: IntervalTrigger)， 向apscheduler的BlockingScheduler對象中添加job,job執行_emit_trigger_instance方法並攜帶參數trigger。
_emit_trigger_instance方法發送消息到隊列上。
具體是調用add_trigger(self, trigger):
1 根據輸入參數，形如:
(Pdb) p trigger
{'uid': u'trigger:core:8c13d9a3-12a3-4cff-88d3-66c804a47590:5e0acee19fa878c4fa07d596f7a893d6', 'parameters': {u'unit': u'minutes', u'delta': 5}, 'ref': u'core.8c13d9a3-12a3-4cff-88d3-66c804a47590', 'name': u'8c13d9a3-12a3-4cff-88d3-66c804a47590', 'type': u'core.st2.IntervalTimer', 'id': '5e78a2f0618fd301a43e2284', 'pack': u'core'}
2 調用_add_job_to_scheduler(self, trigger):
1 根據輸入參數，形如:
(Pdb) p trigger
{'uid': u'trigger:core:8c13d9a3-12a3-4cff-88d3-66c804a47590:5e0acee19fa878c4fa07d596f7a893d6', 'parameters': {u'unit': u'minutes', u'delta': 5}, 'ref': u'core.8c13d9a3-12a3-4cff-88d3-66c804a47590', 'name': u'8c13d9a3-12a3-4cff-88d3-66c804a47590', 'type': u'core.st2.IntervalTimer', 'id': '5e78a2f0618fd301a43e2284', 'pack': u'core'}
2 獲取trigger類型(例如: 'core.st2.IntervalTimer')，參數(例如unit: minutes, delta: 5), 實例化對應apscheduler中trigger對象
(例如: IntervalTrigger, DateTrigger, CronTrigger)
3 調用 _add_job(self, trigger, time_type, replace=True)方法來添加定時任務
1 根據輸入參數，形如:
(Pdb) p trigger
{'uid': u'trigger:core:8c13d9a3-12a3-4cff-88d3-66c804a47590:5e0acee19fa878c4fa07d596f7a893d6', 'parameters': {u'unit': u'minutes', u'delta': 5}, 'ref': u'core.8c13d9a3-12a3-4cff-88d3-66c804a47590', 'name': u'8c13d9a3-12a3-4cff-88d3-66c804a47590', 'type': u'core.st2.IntervalTimer', 'id': '5e78a2f0618fd301a43e2284', 'pack': u'core'}
(Pdb) p time_type
<IntervalTrigger (interval=datetime.timedelta(0, 300), start_date='2020-03-25 17:30:28 CST', timezone='Asia/Shanghai')>
(Pdb) p replace
True
2 向BlockingScheduler對象中添加job,job執行_emit_trigger_instance方法並攜帶參數trigger。
3 _emit_trigger_instance方法發送消息到隊列上，具體是:
_emit_trigger_instance(self, trigger):
1 根據輸入參數，形如:
(Pdb) p trigger
{'uid': u'trigger:core:8c13d9a3-12a3-4cff-88d3-66c804a47590:5e0acee19fa878c4fa07d596f7a893d6', 'parameters': {u'unit': u'minutes', u'delta': 5}, 'ref': u'core.8c13d9a3-12a3-4cff-88d3-66c804a47590', 'name': u'8c13d9a3-12a3-4cff-88d3-66c804a47590', 'type': u'core.st2.IntervalTimer', 'id': '5e78a2f0618fd301a43e2284', 'pack': u'core'}
2 生成payload字典，樣例如下:
{'executed_at': '2020-03-25 10:34:14.085891+00:00', 'schedule': None}
3 構建TraceContext對象(其trace-tag由trigger類型和trigger name拼接而成)，樣例如下:
{'id_': None, 'trace_tag': u'st2.IntervalTimer-8c13d9a3-12a3-4cff-88d3-66c804a47590'}
4 調用TriggerDispatcher類的dispatch(trigger, payload, trace_context=trace_context)方法，具體是:
TriggerDispatcher.dispatch(self, trigger, payload=None, trace_context=None):
1 根據輸入參數，形如:
(Pdb) p trigger
{'uid': u'trigger:core:8c13d9a3-12a3-4cff-88d3-66c804a47590:5e0acee19fa878c4fa07d596f7a893d6', 'parameters': {u'unit': u'minutes', u'delta': 5}, 'ref': u'core.8c13d9a3-12a3-4cff-88d3-66c804a47590', 'name': u'8c13d9a3-12a3-4cff-88d3-66c804a47590', 'type': u'core.st2.IntervalTimer', 'id': '5e78a2f0618fd301a43e2284', 'pack': u'core'}
(Pdb) p payload
{'executed_at': '2020-03-25 10:55:01.848503+00:00', 'schedule': None}
(Pdb) p trace_context
<st2common.models.api.trace.TraceContext object at 0x3be4f90>
(Pdb) p trace_context.__dict__
{'id_': None, 'trace_tag': u'st2.IntervalTimer-8c13d9a3-12a3-4cff-88d3-66c804a47590'}
2 構建payload,形如:
{'trace_context': <st2common.models.api.trace.TraceContext object at 0x3be4f90>, 'trigger': {'uid': u'trigger:core:8c13d9a3-12a3-4cff-88d3-66c804a47590:5e0acee19fa878c4fa07d596f7a893d6', 'parameters': {u'unit': u'minutes', u'delta': 5}, 'ref': u'core.8c13d9a3-12a3-4cff-88d3-66c804a47590', 'name': u'8c13d9a3-12a3-4cff-88d3-66c804a47590', 'type': u'core.st2.IntervalTimer', 'id': '5e78a2f0618fd301a43e2284', 'pack': u'core'}, 'payload': {'executed_at': '2020-03-25 10:55:01.848503+00:00', 'schedule': None}}
3 設置routing_key爲'trigger_instance'
4 調用TriggerInstancePublisher類的publish_trigger(payload=payload, routing_key=routing_key)方法，具體
TriggerInstancePublisher.publish_trigger(self, payload=None, routing_key=None):
s1 根據輸入參數，形如:
(Pdb) p payload
{'trace_context': <st2common.models.api.trace.TraceContext object at 0x3be4f90>, 'trigger': {'uid': u'trigger:core:8c13d9a3-12a3-4cff-88d3-66c804a47590:5e0acee19fa878c4fa07d596f7a893d6', 'parameters': {u'unit': u'minutes', u'delta': 5}, 'ref': u'core.8c13d9a3-12a3-4cff-88d3-66c804a47590', 'name': u'8c13d9a3-12a3-4cff-88d3-66c804a47590', 'type': u'core.st2.IntervalTimer', 'id': '5e78a2f0618fd301a43e2284', 'pack': u'core'}, 'payload': {'executed_at': '2020-03-25 10:55:01.848503+00:00', 'schedule': None}}
(Pdb) p routing_key
'trigger_instance'
s2 調用PoolPublisher類的publish(payload, TRIGGER_INSTANCE_XCHG, routing_key)方法發送消息，具體是:
exchange類型是topic,名稱是: st2.trigger_instances_dispatch
routing_key是'trigger_instance'
4 向jobs字典中添加<trigger的id, job的id>映射
4 返回timer_type，形如:
<IntervalTrigger (interval=datetime.timedelta(0, 300), start_date='2020-03-25 17:30:28 CST', timezone='Asia/Shanghai')>
參考:
stackstorm 2.6代碼