Rendered Template
templates_dict
None
op_args
[]
op_kwargs
{'cluster-configurations': [{'Classification': 'yarn-site',
'Properties': {'yarn.resourcemanager.am.max-attempts': '1'}}],
'cluster-name': 'exchange-trade-summary-backfill_all',
'core-instance-capacity': 20,
'core-instance-types': 'm5.2xlarge,m5.4xlarge',
'dag-id': 'exchange-trade-summary-backfill_all_v002',
'ebs-volume-size': '200',
'emr-steps': '[\n'
' {\n'
' "step-name": "SupplyDenormalizationJobBackfill",\n'
' "config-json": [\n'
' {"spark.driver.memory":"20g"},\n'
' {"spark.driver.cores":"5"},\n'
' '
'{"spark.serializer":"org.apache.spark.serializer.KryoSerializer"},\n'
' {"spark.task.maxFailures":"20"},\n'
' {"spark.yarn.maxAppAttempts":"10"},\n'
' {"spark.stage.maxConsecutiveAttempts":"20"}\n'
' ],\n'
' "main-class": '
'"com.viooh.smex.supply.denormalization.SupplyDataDenormalizationBackfill",\n'
' "group-id":"com/viooh/smex",\n'
' "artifact": "supply-denormalization",\n'
' "jars": "/usr/lib/spark/external/lib/spark-avro.jar"\n'
' },\n'
' {\n'
' "step-name": "DemandDenormalizationJobBackfill",\n'
' "config-json": [\n'
' {"spark.driver.memory":"20g"},\n'
' {"spark.driver.cores":"5"},\n'
' '
'{"spark.serializer":"org.apache.spark.serializer.KryoSerializer"},\n'
' {"spark.task.maxFailures":"20"},\n'
' {"spark.yarn.maxAppAttempts":"10"},\n'
' {"spark.stage.maxConsecutiveAttempts":"20"}\n'
' ],\n'
' "main-class": '
'"com.viooh.smex.demand.denormalization.DemandDataDenormalizationBackfill",\n'
' "group-id":"com/viooh/smex",\n'
' "artifact": "demand-denormalization",\n'
' "jars": "/usr/lib/spark/external/lib/spark-avro.jar"\n'
' },\n'
' {\n'
' "step-name": "DealsyncDenormalizationJobBackfill",\n'
' "config-json": [\n'
' {"spark.driver.memory":"20g"},\n'
' {"spark.driver.cores":"5"},\n'
' '
'{"spark.serializer":"org.apache.spark.serializer.KryoSerializer"},\n'
' {"spark.task.maxFailures":"20"},\n'
' {"spark.yarn.maxAppAttempts":"10"},\n'
' {"spark.stage.maxConsecutiveAttempts":"20"}\n'
' ],\n'
' "main-class": '
'"com.viooh.smex.dealsync.denormalization.DealSyncDenormalizationBackfill",\n'
' "group-id":"com/viooh/smex",\n'
' "artifact": "dealsync-denormalization"\n'
' },\n'
' {\n'
' "step-name": "ModerationDenormalizationJobBackfill",\n'
' "config-json": [\n'
' {"spark.driver.memory":"20g"},\n'
' {"spark.driver.cores":"5"},\n'
' '
'{"spark.serializer":"org.apache.spark.serializer.KryoSerializer"},\n'
' {"spark.task.maxFailures":"20"},\n'
' {"spark.yarn.maxAppAttempts":"10"},\n'
' {"spark.stage.maxConsecutiveAttempts":"20"}\n'
' ],\n'
' "main-class": '
'"com.viooh.smex.moderation.denormalization.ModerationDenormalizationBackfill",\n'
' "group-id":"com/viooh/smex",\n'
' "artifact": "moderation-denormalization",\n'
' "jars": "/usr/lib/spark/external/lib/spark-avro.jar"\n'
' },\n'
' {\n'
' "step-name": "DealDenormalizationJobBackfill",\n'
' "config-json": [\n'
' {"spark.driver.memory":"20g"},\n'
' {"spark.driver.cores":"5"},\n'
' '
'{"spark.serializer":"org.apache.spark.serializer.KryoSerializer"},\n'
' {"spark.task.maxFailures":"20"},\n'
' {"spark.yarn.maxAppAttempts":"10"},\n'
' {"spark.stage.maxConsecutiveAttempts":"20"}\n'
' ],\n'
' "main-class": '
'"com.viooh.smex.deal.denormalization.DealDenormalizationBackfill",\n'
' "group-id":"com/viooh/smex",\n'
' "artifact": "deal-denormalization",\n'
' "jars": "/usr/lib/spark/external/lib/spark-avro.jar"\n'
' },\n'
' {\n'
' "step-name": "ExchangeTradeSummaryJobBackfill",\n'
' "config-json": [\n'
' {"spark.driver.memory":"16g"},\n'
' '
'{"spark.serializer":"org.apache.spark.serializer.KryoSerializer"},\n'
' {"spark.task.maxFailures":"20"},\n'
' {"spark.yarn.maxAppAttempts":"10"},\n'
' {"spark.stage.maxConsecutiveAttempts":"20"}\n'
' ],\n'
' "main-class": '
'"com.viooh.smex.trade.summary.ExchangeTradeSummaryBackfill",\n'
' "group-id":"com/viooh/smex",\n'
' "artifact": "exchange-trade-summary"\n'
' },\n'
' {\n'
' "step-name": '
'"ExportVioohHourLevelReportToDbV2Backfill",\n'
' '
'"jar-location":"s3://elasticmapreduce/libs/script-runner/script-runner.jar",\n'
' '
'"script-file":"scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh",\n'
' "script-args": '
'"$ENV#0.0.2#db-import-viooh-hourly-ui-report-v2#backfill-frame#$EXECUTION_DATETIME#$ARTIFACT_BUCKET#scripts/exchange-trade-summary/sspui_reporting#backfill-frame#cn-northwest-1"\n'
' },\n'
' {\n'
' "step-name": '
'"ExportVioohHourLevelReportToDbV2Backfill",\n'
' '
'"jar-location":"s3://elasticmapreduce/libs/script-runner/script-runner.jar",\n'
' '
'"script-file":"scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh",\n'
' "script-args": '
'"$ENV#0.0.2#db-import-viooh-hourly-creative-report#backfill-creative#$EXECUTION_DATETIME#$ARTIFACT_BUCKET#scripts/exchange-trade-summary/sspui_reporting#backfill-creative#cn-northwest-1"\n'
' }\n'
']',
'master-instance-types': 'm5.2xlarge,m5.4xlarge',
'schedule_interval': None,
'task-instance-capacity': 0,
'task-instance-types': 'm5.2xlarge,m5.4xlarge',
'trigger_dags': []}