Rendered Template
templates_dict
None
op_args
[]
op_kwargs
{'cluster-configurations': [{'Classification': 'yarn-site',
'Properties': {'yarn.resourcemanager.am.max-attempts': '2'}}],
'cluster-name': 'exchange-trade-summary_v1',
'core-instance-capacity': 5,
'core-instance-types': 'm5.xlarge,m5.2xlarge,m5.4xlarge',
'dag-id': 'exchange-trade-summary_v1_v002',
'emr-steps': '[\n'
' {\n'
' "step-name": "SupplyDenormalizationJob",\n'
' "config-json": [\n'
' {"spark.driver.memory":"6g"}\n'
' ],\n'
' "main-class": '
'"com.viooh.smex.supply.denormalization.SupplyDataDenormalization",\n'
' "group-id":"com/viooh/smex",\n'
' "artifact": "supply-denormalization"\n'
' },\n'
' {\n'
' "step-name": "DemandDenormalizationJob",\n'
' "config-json": [\n'
' {"spark.driver.memory":"6g"}\n'
' ],\n'
' "main-class": '
'"com.viooh.smex.demand.denormalization.DemandDataDenormalization",\n'
' "group-id":"com/viooh/smex",\n'
' "artifact": "demand-denormalization"\n'
' },\n'
' {\n'
' "step-name": "DealSyncDenormalizationJob",\n'
' "config-json": [\n'
' {"spark.driver.memory":"6g"}\n'
' ],\n'
' "main-class": '
'"com.viooh.smex.dealsync.denormalization.DealSyncDenormalization",\n'
' "group-id":"com/viooh/smex",\n'
' "artifact": "dealsync-denormalization"\n'
' },\n'
' {\n'
' "step-name": "ModerationDenormalizationJob",\n'
' "config-json": [\n'
' {"spark.driver.memory":"6g"}\n'
' ],\n'
' "main-class": '
'"com.viooh.smex.moderation.denormalization.ModerationDenormalization",\n'
' "group-id":"com/viooh/smex",\n'
' "artifact": "moderation-denormalization",\n'
' "jars": "/usr/lib/spark/external/lib/spark-avro.jar"\n'
' },\n'
' {\n'
' "step-name": "DealDenormalizationJob",\n'
' "config-json": [\n'
' {"spark.driver.memory":"6g"}\n'
' ],\n'
' "main-class": '
'"com.viooh.smex.deal.denormalization.DealDenormalization",\n'
' "group-id":"com/viooh/smex",\n'
' "artifact": "deal-denormalization",\n'
' "jars": "/usr/lib/spark/external/lib/spark-avro.jar"\n'
' },\n'
' {\n'
' "step-name": "ExchangeTradeSummaryJob",\n'
' "config-json": [\n'
' {"spark.driver.memory":"6g"},\n'
' '
'{"spark.serializer":"org.apache.spark.serializer.KryoSerializer"},\n'
' {"spark.task.maxFailures":"20"},\n'
' {"spark.yarn.maxAppAttempts":"10"},\n'
' {"spark.stage.maxConsecutiveAttempts":"20"},\n'
' {"spark.app.config.key":"exchange-trade-summary"}\n'
' ],\n'
' "main-class": '
'"com.viooh.smex.trade.summary.ExchangeTradeSummary",\n'
' "group-id":"com/viooh/smex",\n'
' "artifact": "exchange-trade-summary"\n'
' },\n'
' {\n'
' "step-name": "CopySmartExchangeMediaOwnerReport",\n'
' '
'"jar-location":"s3://elasticmapreduce/libs/script-runner/script-runner.jar",\n'
' '
'"script-file":"scripts/exchange-trade-summary/1.3.0/job.sh",\n'
' "script-args": '
'"$ENV#$NEXT_EXECUTION_DATE#$NEXT_EXECUTION_HOUR#mediaowner-trade-report#trade-summary-reports#v1#csv#cn-northwest-1"\n'
' },\n'
' {\n'
' "step-name": "CopyExchangeLiteMediaOwnerReport",\n'
' '
'"jar-location":"s3://elasticmapreduce/libs/script-runner/script-runner.jar",\n'
' '
'"script-file":"scripts/exchange-trade-summary/1.3.0/job.sh",\n'
' "script-args": '
'"$ENV#$EXECUTION_DATE#$EXECUTION_HOUR#exchange-lite-mediaowner-trade-report#trade-summary-reports#v1#csv#cn-northwest-1"\n'
' },\n'
' {\n'
' "step-name": '
'"CopySmartExchangeFrameLevelMediaOwnerReport",\n'
' '
'"jar-location":"s3://elasticmapreduce/libs/script-runner/script-runner.jar",\n'
' '
'"script-file":"scripts/exchange-trade-summary/1.3.0/job.sh",\n'
' "script-args": '
'"$ENV#$NEXT_EXECUTION_DATE#$NEXT_EXECUTION_HOUR#frame-level-mediaowner-trade-report#frame-level-media-owner-reports#v1#csv#cn-northwest-1"\n'
' },\n'
' {\n'
' "step-name": '
'"CopyExchangeLiteFrameLevelMediaOwnerReport",\n'
' '
'"jar-location":"s3://elasticmapreduce/libs/script-runner/script-runner.jar",\n'
' '
'"script-file":"scripts/exchange-trade-summary/1.3.0/job.sh",\n'
' "script-args": '
'"$ENV#$EXECUTION_DATE#$EXECUTION_HOUR#frame-level-exchange-lite-mediaowner-trade-report#frame-level-media-owner-reports#v1#csv#cn-northwest-1"\n'
' },\n'
' {\n'
' "step-name": "CopySmartExchangeMediaOwnerReportV2",\n'
' '
'"jar-location":"s3://elasticmapreduce/libs/script-runner/script-runner.jar",\n'
' '
'"script-file":"scripts/exchange-trade-summary/1.3.0/job.sh",\n'
' "script-args": '
'"$ENV#$NEXT_EXECUTION_DATE#$NEXT_EXECUTION_HOUR#mediaowner-trade-report#trade-summary-reports#v2#json#cn-northwest-1"\n'
' },\n'
' {\n'
' "step-name": "CopyExchangeLiteMediaOwnerReportV2",\n'
' '
'"jar-location":"s3://elasticmapreduce/libs/script-runner/script-runner.jar",\n'
' '
'"script-file":"scripts/exchange-trade-summary/1.3.0/job.sh",\n'
' "script-args": '
'"$ENV#$EXECUTION_DATE#$EXECUTION_HOUR#exchange-lite-mediaowner-trade-report#trade-summary-reports#v2#json#cn-northwest-1"\n'
' },\n'
' {\n'
' "step-name": "ExportVioohHourLevelReportToDbV2",\n'
' '
'"jar-location":"s3://elasticmapreduce/libs/script-runner/script-runner.jar",\n'
' '
'"script-file":"scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh",\n'
' "script-args": '
'"$ENV#0.0.2#db-import-viooh-hourly-ui-report-v2#1#$EXECUTION_DATETIME#$ARTIFACT_BUCKET#scripts/exchange-trade-summary/sspui_reporting#deal#cn-northwest-1"\n'
' },\n'
' {\n'
' "step-name": '
'"ExportVioohHourlyCreativeLevelReportToDb",\n'
' '
'"jar-location":"s3://elasticmapreduce/libs/script-runner/script-runner.jar",\n'
' '
'"script-file":"scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh",\n'
' "script-args": '
'"$ENV#0.0.2#db-import-viooh-hourly-creative-report#1#$EXECUTION_DATETIME#$ARTIFACT_BUCKET#scripts/exchange-trade-summary/sspui_reporting#creative#cn-northwest-1"\n'
' }\n'
']',
'emr-version': 'emr-5.35.0',
'master-instance-types': 'm5.xlarge,m5.2xlarge,m5.4xlarge',
'schedule_interval': '@hourly',
'task-instance-capacity': 0,
'task-instance-types': 'm5.xlarge,m5.2xlarge,m5.4xlarge',
'trigger_dags': []}