Rendered Template
templates_dict
None
op_args
[]
op_kwargs
{'cluster-name': 'livepop-shuffle-aggregator-modeltransform',
'core-instance-capacity': 3,
'core-instance-types': 'm5.xlarge,m5.2xlarge',
'dag-id': 'livepop-shuffle-aggregator-modeltransform_v004',
'ebs-volume-size': '50',
'emr-steps': '[\n'
' {\n'
' "step-name": "LivePOPShuffle",\n'
' "config-json": [\n'
' {"spark.driver.memory":"9g"},\n'
' '
'{"spark.serializer":"org.apache.spark.serializer.KryoSerializer"}\n'
' ],\n'
' "main-class": '
'"com.viooh.pop.data.live.shuffle.RawLivePOPShuffleMain",\n'
' "group-id":"com/viooh/pop",\n'
' "artifact": "pop-shuffle-live",\n'
' "jars": "/usr/lib/spark/external/lib/spark-avro.jar",\n'
' "enable-custom-metrics" : "True"\n'
' },\n'
' {\n'
' "step-name": "PopAggregator",\n'
' "config-json": [\n'
' {"spark.driver.memory":"9g"},\n'
' '
'{"spark.serializer":"org.apache.spark.serializer.KryoSerializer"}\n'
' ],\n'
' "main-class": '
'"com.viooh.pop.aggregator.livepop.LivePOPAggregatorMain",\n'
' "group-id":"com/viooh/pop",\n'
' "artifact": "pop-data-aggregator",\n'
' "enable-custom-metrics" : "True"\n'
' },\n'
' {\n'
' "step-name": "ModelTransform",\n'
' "config-json": [\n'
' {"spark.driver.memory":"9g"}\n'
' ],\n'
' "main-class": '
'"uk.co.viooh.job.modeltransform.ModelTransform",\n'
' "group-id": "uk/co/viooh",\n'
' "artifact": "pandora-model-transform",\n'
' "enable-custom-metrics" : "True"\n'
' }\n'
']',
'emr-version': 'emr-5.35.0',
'input-validation-step-config': 'livepop_shuffle_job_validation_config',
'master-instance-types': 'm5.xlarge,m5.2xlarge',
'schedule_interval': '0 * * * *',
'task-instance-capacity': 0,
'task-instance-types': 'm5.xlarge,m5.2xlarge',
'trigger_dags': []}