op_kwargs |
{'master-instance-types': 'm5.xlarge,m5.2xlarge,m5.4xlarge', 'core-instance-types': 'm5.xlarge,m5.2xlarge,m5.4xlarge', 'task-instance-types': 'm5.xlarge,m5.2xlarge,m5.4xlarge', 'core-instance-capacity': 5, 'task-instance-capacity': 0, 'emr-version': 'emr-5.35.0', 'cluster-configurations': [{'Classification': 'yarn-site', 'Properties': {'yarn.resourcemanager.am.max-attempts': '2'}}], 'emr-steps': '[\n {\n "step-name": "SupplyDenormalizationJob",\n "config-json": [\n {"spark.driver.memory":"6g"}\n ],\n "main-class": "com.viooh.smex.supply.denormalization.SupplyDataDenormalization",\n "artifact": "supply-denormalization"\n },\n {\n "step-name": "DemandDenormalizationJob",\n "config-json": [\n {"spark.driver.memory":"6g"}\n ],\n "main-class": "com.viooh.smex.demand.denormalization.DemandDataDenormalization",\n "artifact": "demand-denormalization"\n },\n {\n "step-name": "DealSyncDenormalizationJob",\n "config-json": [\n {"spark.driver.memory":"6g"}\n ],\n "main-class": "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization",\n "artifact": "dealsync-denormalization"\n },\n {\n "step-name": "ModerationDenormalizationJob",\n "config-json": [\n {"spark.driver.memory":"6g"}\n ],\n "main-class": "com.viooh.smex.moderation.denormalization.ModerationDenormalization",\n "artifact": "moderation-denormalization",\n "jars": "/usr/lib/spark/external/lib/spark-avro.jar"\n },\n {\n "step-name": "DealDenormalizationJob",\n "config-json": [\n {"spark.driver.memory":"6g"}\n ],\n "main-class": "com.viooh.smex.deal.denormalization.DealDenormalization",\n "artifact": "deal-denormalization",\n "jars": "/usr/lib/spark/external/lib/spark-avro.jar"\n },\n {\n "step-name": "ExchangeTradeSummaryJob",\n "config-json": [\n {"spark.driver.memory":"6g"},\n {"spark.serializer":"org.apache.spark.serializer.KryoSerializer"},\n {"spark.task.maxFailures":"20"},\n {"spark.yarn.maxAppAttempts":"10"},\n {"spark.stage.maxConsecutiveAttempts":"20"},\n {"spark.app.config.key":"exchange-trade-summary"}\n ],\n "main-class": "com.viooh.smex.trade.summary.ExchangeTradeSummary",\n "artifact": "exchange-trade-summary"\n },\n {\n "step-name": "CopySmartExchangeMediaOwnerReport",\n "jar-location":"s3://elasticmapreduce/libs/script-runner/script-runner.jar",\n "script-file":"scripts/exchange-trade-summary/1.3.0/job.sh",\n "script-args": "$ENV#$NEXT_EXECUTION_DATE#$NEXT_EXECUTION_HOUR#mediaowner-trade-report#trade-summary-reports#v1#csv"\n },\n {\n "step-name": "CopyExchangeLiteMediaOwnerReport",\n "jar-location":"s3://elasticmapreduce/libs/script-runner/script-runner.jar",\n "script-file":"scripts/exchange-trade-summary/1.3.0/job.sh",\n "script-args": "$ENV#$EXECUTION_DATE#$EXECUTION_HOUR#exchange-lite-mediaowner-trade-report#trade-summary-reports#v1#csv"\n },\n {\n "step-name": "CopySmartExchangeFrameLevelMediaOwnerReport",\n "jar-location":"s3://elasticmapreduce/libs/script-runner/script-runner.jar",\n "script-file":"scripts/exchange-trade-summary/1.3.0/job.sh",\n "script-args": "$ENV#$NEXT_EXECUTION_DATE#$NEXT_EXECUTION_HOUR#frame-level-mediaowner-trade-report#frame-level-media-owner-reports#v1#csv"\n },\n {\n "step-name": "CopyExchangeLiteFrameLevelMediaOwnerReport",\n "jar-location":"s3://elasticmapreduce/libs/script-runner/script-runner.jar",\n "script-file":"scripts/exchange-trade-summary/1.3.0/job.sh",\n "script-args": "$ENV#$EXECUTION_DATE#$EXECUTION_HOUR#frame-level-exchange-lite-mediaowner-trade-report#frame-level-media-owner-reports#v1#csv"\n },\n {\n "step-name": "CopySmartExchangeMediaOwnerReportV2",\n "jar-location":"s3://elasticmapreduce/libs/script-runner/script-runner.jar",\n "script-file":"scripts/exchange-trade-summary/1.3.0/job.sh",\n "script-args": "$ENV#$NEXT_EXECUTION_DATE#$NEXT_EXECUTION_HOUR#mediaowner-trade-report#trade-summary-reports#v2#json"\n },\n {\n "step-name": "CopyExchangeLiteMediaOwnerReportV2",\n "jar-location":"s3://elasticmapreduce/libs/script-runner/script-runner.jar",\n "script-file":"scripts/exchange-trade-summary/1.3.0/job.sh",\n "script-args": "$ENV#$EXECUTION_DATE#$EXECUTION_HOUR#exchange-lite-mediaowner-trade-report#trade-summary-reports#v2#json"\n },\n {\n "step-name": "ExportVioohHourLevelReportToDbV2",\n "jar-location":"s3://elasticmapreduce/libs/script-runner/script-runner.jar",\n "script-file":"scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh",\n "script-args": "$ENV#0.0.2#db-import-viooh-hourly-ui-report-v2#1#$EXECUTION_DATETIME#$ARTIFACT_BUCKET#scripts/exchange-trade-summary/sspui_reporting#deal#cn-northwest-1"\n },\n {\n "step-name": "ExportVioohHourlyCreativeLevelReportToDb",\n "jar-location":"s3://elasticmapreduce/libs/script-runner/script-runner.jar",\n "script-file":"scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh",\n "script-args": "$ENV#0.0.2#db-import-viooh-hourly-creative-report#1#$EXECUTION_DATETIME#$ARTIFACT_BUCKET#scripts/exchange-trade-summary/sspui_reporting#creative#cn-northwest-1"\n }\n]', 'cluster-name': 'exchange-trade-summary', 'dag-id': 'exchange-trade-summary_v002', 'schedule_interval': '@hourly', 'trigger_dags': []} |