Rendered Template
templates_dict
None
op_args
[]
op_kwargs
{'audience-config-file': 'primary_audience_global_conf',
'cluster-name': 'primary-audience-global-JCDECAUX_ES-3000',
'core-instance-capacity': 5,
'core-instance-types': 'm5.2xlarge,m5.4xlarge',
'dag-id': 'primary-audience-global_v003',
'emr-steps': '[{\n'
' "step-name": "PrimaryAudienceGlobal-integrity",\n'
' "py-script": "jobs/integrity_job.py",\n'
' "config-file": '
'"configs/primary-audience-global/1.0.0/job.json",\n'
' "config-json": [\n'
' {"spark.app.audience.country":"ES"},\n'
' {"spark.app.audience.media.owner":"JCDECAUX_ES"},\n'
' {"spark.app.audience.version":"3000"},\n'
' {"spark.app.audience.process.dynamic":"True"},\n'
' {"spark.app.audience.process.static":"False"},\n'
' '
'{"spark.app.audience.bucket.name":"s3a://viooh-datashare-jcd-es"},\n'
' '
'{"spark.app.audience.digital.file":"jcd-es-data/r12/dynamic"},\n'
' {"spark.app.audience.static.file":""},\n'
' '
'{"spark.app.audience.demographic.file":"jcd-es-data/r12/demographic"},\n'
' {"spark.app.audience.check.demo":"True"}\n'
' ],\n'
' "artifact": "AudiencesPipeline"\n'
' },\n'
' {\n'
' "step-name": "PrimaryAudienceGlobal-load",\n'
' "py-script": "jobs/etl_job.py",\n'
' "config-file": '
'"configs/primary-audience-global/1.0.0/job.json",\n'
' "config-json": [\n'
' {"spark.app.audience.country":"ES"},\n'
' {"spark.app.audience.media.owner":"JCDECAUX_ES"},\n'
' {"spark.app.audience.version":"3000"},\n'
' {"spark.app.audience.process.dynamic":"True"},\n'
' {"spark.app.audience.process.static":"False"},\n'
' {"spark.app.audience.category.group.name":"ES"},\n'
' {"spark.app.audience.category.group.id":"001"}\n'
' ],\n'
' "artifact": "AudiencesPipeline"\n'
'},{\n'
' "step-name": "RegenerateRoute",\n'
' "config-file": '
'"configs/primary-audience-global-cassandra/1.0.0/job.json",\n'
' "config-json": [\n'
' '
'{"spark.app.primary.audience.base.path": '
'"s3a://global-audiences-output/ES/JCDECAUX_ES/3000/raw/%date%/dynamic/"},\n'
' {"spark.app.primary.audience.date": '
'"20201214"},\n'
' '
'{"spark.app.primary.audience.cassandra.keyspace": '
'"es_audience"},\n'
' {"spark.app.primary.audience.version": '
'"3000"}\n'
' ],\n'
' "artifact": "RegenerateRoute",\n'
' "packages": '
'"com.datastax.spark:spark-cassandra-connector_2.11:2.3.0",\n'
' "py-script": '
'"com/viooh/routegen/primary_audience_loader_cassandra.py"\n'
' },{\n'
' "step-name": "SaveToDataBase",\n'
' "config-file": '
'"configs/primary-audience-global-masterdb-uat/1.0.0/job.json",\n'
' "config-json": [\n'
' {"spark.route.version": "3000"},\n'
' '
'{"spark.location.outputFoldermdb":"s3a://global-audiences-output/ES/JCDECAUX_ES/3000/"},\n'
' '
'{"spark.external.db.database":"Espain_uat"},\n'
' '
'{"spark.date.daytouplaodtomysql":"20201214"}\n'
' ],\n'
' "main-class": '
'"com.viooh.restoremasterdb.SaveToDataBase",\n'
' "artifact": "restoremasterdb_2.11",\n'
' "packages": "mysql:mysql-connector-java:5.1.46"\n'
' }]',
'emr-version': 'emr-5.21.0',
'job-type': 'batch',
'master-instance-types': 'm5.2xlarge,m5.4xlarge',
'schedule_interval': None,
'task-instance-capacity': 0,
'task-instance-types': 'm5.2xlarge,m5.4xlarge',
'trigger_dags': []}