Skip to content

Commit c42be84

Browse files
LearningJournalLearningJournal
committed
Initial Commit
1 parent 049167c commit c42be84

40 files changed

+1414
-0
lines changed

Diff for: ‎SBDL - Starter/.gitignore

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Default ignored files
2+
/workspace.xml
3+
.idea/*
4+
lib/__pycache__/*
5+

Diff for: ‎SBDL - Starter/Jenkinsfile

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
pipeline {
2+
agent any
3+
4+
stages {
5+
stage('Build') {
6+
steps {
7+
sh 'pipenv --python python3 sync'
8+
}
9+
}
10+
stage('Test') {
11+
steps {
12+
sh 'pipenv run pytest'
13+
}
14+
}
15+
stage('Package') {
16+
when{
17+
anyOf{ branch "master" ; branch 'release' }
18+
}
19+
steps {
20+
sh 'zip -r sbdl.zip lib'
21+
}
22+
}
23+
stage('Release') {
24+
when{
25+
branch 'release'
26+
}
27+
steps {
28+
sh "scp -i /home/prashant/cred/edge-node_key.pem -o 'StrictHostKeyChecking no' -r sbdl.zip log4j.properties sbdl_main.py sbdl_submit.sh conf prashant@40.117.123.105:/home/prashant/sbdl-qa"
29+
}
30+
}
31+
stage('Deploy') {
32+
when{
33+
branch 'master'
34+
}
35+
steps {
36+
sh "scp -i /home/prashant/cred/edge-node_key.pem -o 'StrictHostKeyChecking no' -r sbdl.zip log4j.properties sbdl_main.py sbdl_submit.sh conf prashant@40.117.123.105:/home/prashant/sbdl-prod"
37+
}
38+
}
39+
}
40+
}

Diff for: ‎SBDL - Starter/Pipfile

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
[[source]]
2+
url = "https://pypi.org/simple"
3+
verify_ssl = true
4+
name = "pypi"
5+
6+
[packages]
7+
pyspark = "*"
8+
pytest = "*"
9+
10+
[dev-packages]
11+
12+
[requires]
13+
python_version = "3.10"
14+

Diff for: ‎SBDL - Starter/conf/sbdl.conf

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
[LOCAL]
2+
enable.hive = false
3+
hive.database = null
4+
kafka.topic = sbdl_kafka_cloud
5+
[QA]
6+
enable.hive = true
7+
hive.database = sbdl_db_qa
8+
kafka.topic = sbdl_kafka_qa
9+
[PROD]
10+
enable.hive = true
11+
hive.database = sbdl_db
12+
kafka.topic = sbdl_kafka
13+

Diff for: ‎SBDL - Starter/conf/spark.conf

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
[LOCAL]
2+
spark.app.name = sbdl-local
3+
spark.executor.instances = 2
4+
spark.executor.cores = 1
5+
spark.executor.memory = 1G
6+
spark.sql.shuffle.partitions = 5
7+
[QA]
8+
spark.app.name = sbdl-qa
9+
spark.executor.instances = 2
10+
spark.executor.cores = 1
11+
spark.executor.memory = 4G
12+
spark.sql.shuffle.partitions = 1000
13+
[PROD]
14+
spark.app.name = sbdl
15+
spark.executor.instances = 2
16+
spark.executor.cores = 1
17+
spark.executor.memory = 4G
18+
spark.sql.shuffle.partitions = 1000
19+
20+

Diff for: ‎SBDL - Starter/lib/Utils.py

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
from pyspark.sql import SparkSession
2+
3+
4+
def get_spark_session(env):
5+
if env == "LOCAL":
6+
return SparkSession.builder \
7+
.config('spark.driver.extraJavaOptions',
8+
'-Dlog4j.configuration=file:log4j.properties') \
9+
.master("local[2]") \
10+
.enableHiveSupport() \
11+
.getOrCreate()
12+
else:
13+
return SparkSession.builder \
14+
.enableHiveSupport() \
15+
.getOrCreate()
16+
17+

Diff for: ‎SBDL - Starter/lib/__init__.py

Whitespace-only changes.

Diff for: ‎SBDL - Starter/lib/logger.py

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
class Log4j(object):
2+
def __init__(self, spark):
3+
log4j = spark._jvm.org.apache.log4j
4+
self.logger = log4j.LogManager.getLogger("sbdl")
5+
6+
def warn(self, message):
7+
self.logger.warn(message)
8+
9+
def info(self, message):
10+
self.logger.info(message)
11+
12+
def error(self, message):
13+
self.logger.error(message)
14+
15+
def debug(self, message):
16+
self.logger.debug(message)
17+
18+

Diff for: ‎SBDL - Starter/log4j.properties

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Set everything to be logged to the console
2+
log4j.rootCategory=WARN, console
3+
4+
# define console appender
5+
log4j.appender.console=org.apache.log4j.ConsoleAppender
6+
log4j.appender.console.target=System.out
7+
log4j.appender.console.layout=org.apache.log4j.PatternLayout
8+
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
9+
10+
#application log
11+
log4j.logger.sbdl=INFO, console, file
12+
log4j.additivity.sbdl=false
13+
14+
#define following in Java System
15+
# -Dlog4j.configuration=file:log4j.properties
16+
17+
# Recommendations from Spark template
18+
log4j.logger.org.apache.spark.repl.Main=WARN
19+
log4j.logger.org.spark_project.jetty=WARN
20+
log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR
21+
log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
22+
log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
23+
log4j.logger.org.apache.parquet=ERROR
24+
log4j.logger.parquet=ERROR
25+
log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
26+
log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR
27+

Diff for: ‎SBDL - Starter/sbdl_main.py

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import sys
2+
3+
from lib import Utils
4+
from lib.logger import Log4j
5+
6+
if __name__ == '__main__':
7+
8+
if len(sys.argv) < 3:
9+
print("Usage: sbdl {local, qa, prod} {load_date} : Arguments are missing")
10+
sys.exit(-1)
11+
12+
job_run_env = sys.argv[1].upper()
13+
load_date = sys.argv[2]
14+
15+
spark = Utils.get_spark_session(job_run_env)
16+
logger = Log4j(spark)
17+
18+
logger.info("Finished creating Spark Session")

Diff for: ‎SBDL - Starter/sbdl_submit.sh

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
spark-submit --master yarn --deploy-mode cluster \
2+
--py-files sdbl_lib.zip \
3+
--files conf/sdbl.conf,conf/spark.conf,log4j.properties \
4+
sbdl_main.py qa 2022-08-02
5+
6+
7+

Diff for: ‎SBDL - Starter/test_data/accounts/account_samples.csv

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
load_date,active_ind,account_id,source_sys,account_start_date,legal_title_1,legal_title_2,tax_id_type,tax_id,branch_code,country
2+
2022-08-02,1,6982391060,COH,2018-03-24T13:56:45.000+05:30,Tiffany Riley,Matthew Davies,EIN,ZLCK91795330413525,ACXMGBA5,Mexico
3+
2022-08-02,1,6982391061,ADS,2018-07-19T11:24:49.000+05:30,Garcia and Sons,Taylor Guzman,SSP,CADU39916151090321,SHJFGBML,United States
4+
2022-08-02,1,6982391067,BDL,2018-08-29T17:18:59.000+05:30,Acosta Inc,David Walker,SSP,UJLN20870916345792,WZTEGBTG,Canada
5+
2022-08-02,0,6982391063,ADS,2019-02-15T09:46:10.000+05:30,Christopher Moreno,,CPR,APVN98456428071508,KRGZGB9S,Canada
6+
2022-08-02,1,6982391064,ADS,2018-03-28T15:47:43.000+05:30,Allen Group,,CPR,WJMX61093523376960,OCCKGB65,Canada
7+
2022-08-02,1,6982391065,COR,2018-06-30T22:33:19.000+05:30,Austin Miles,,EIN,TYAB75470638120665,XCVKGB49,Canada
8+
2022-08-02,1,6982391066,BDL,2017-08-25T03:00:00.000+05:30,Miss Lisa Lee,,SSP,WFDP61047142851240,CVYEGBJC,United States
9+
2022-08-02,1,6982391062,CML,2017-09-24T08:14:17.000+05:30,Theresa Mays,,CPR,WTUP76582369402245,XCVKGB49,Canada
10+
2022-08-02,1,6982391068,BDL,2018-05-11T15:24:57.000+05:30,Amanda Martin,,EIN,JDPX30428146546118,ACXMGBA5,Mexico

Diff for: ‎SBDL - Starter/test_data/parties/party_samples.csv

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
load_date,account_id,party_id,relation_type,relation_start_date
2+
2022-08-02,6982391060,9823462810,F-N,2019-07-29T06:21:32.000+05:30
3+
2022-08-02,6982391061,9823462811,F-N,2018-08-31T05:27:22.000+05:30
4+
2022-08-02,6982391062,9823462812,F-N,2018-08-25T15:50:29.000+05:30
5+
2022-08-02,6982391063,9823462813,F-N,2018-05-11T07:23:28.000+05:30
6+
2022-08-02,6982391064,9823462814,F-N,2019-06-06T14:18:12.000+05:30
7+
2022-08-02,6982391065,9823462815,F-N,2019-05-04T05:12:37.000+05:30
8+
2022-08-02,6982391066,9823462816,F-N,2019-05-15T10:39:29.000+05:30
9+
2022-08-02,6982391067,9823462817,F-N,2018-05-16T09:53:04.000+05:30
10+
2022-08-02,6982391068,9823462818,F-N,2017-11-27T01:20:12.000+05:30
11+
2022-08-02,6982391067,9823462820,F-S,2017-11-20T14:18:05.000+05:30
12+
2022-08-02,6982391067,9823462821,F-S,2018-07-19T18:56:57.000+05:30

Diff for: ‎SBDL - Starter/test_data/party_address/address_samples.csv

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
load_date,party_id,address_line_1,address_line_2,city,postal_code,country_of_address,address_start_date
2+
2022-08-02,9823462810,45229 Drake Route,13306 Corey Point,Shanefort,77163,Canada,2019-02-26
3+
2022-08-02,9823462811,361 Robinson Green,3511 Rebecca Mission,North Tyler,34118,Canada,2018-01-28
4+
2022-08-02,9823462812,039 Daniel Mount,8219 Hernandez Lodge Suite 875,Boltonborough,71648,Mexico,2018-12-07
5+
2022-08-02,9823462813,05550 Nancy Rapids,9471 Zachary Canyon,East Davidport,02504,United States,2019-04-02
6+
2022-08-02,9823462814,5227 Wagner Pines,189 Julie Throughway,West Amanda,78962,Canada,2018-07-11
7+
2022-08-02,9823462815,6993 Diane Alley,8945 Trevor Greens,Kendrafurt,50790,United States,2017-10-08
8+
2022-08-02,9823462816,23450 Timothy Divide,125 Johnson Mountain Suite 701,Osbornetown,04756,Canada,2018-11-28
9+
2022-08-02,9823462817,251 Lee Tunnel,09795 Tara Station Suite 264,New Michelleborough,05505,United States,2019-04-20
10+
2022-08-02,9823462818,7537 Clarke Club,74089 Jerry Trail,Hunterville,19596,United States,2018-07-17

Diff for: ‎SBDL - Starter/test_data/results/final_df.json

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{"eventHeader":{"eventIdentifier":"c361a145-d2fc-434e-a608-9688caa6d22e","eventType":"SBDL-Contract","majorSchemaVersion":1,"minorSchemaVersion":0,"eventDateTime":"2022-09-06T20:49:03+0530"},"keys":[{"keyField":"contractIdentifier","keyValue":"6982391060"}],"payload":{"contractIdentifier":{"operation":"INSERT","newValue":"6982391060"},"sourceSystemIdentifier":{"operation":"INSERT","newValue":"COH"},"contactStartDateTime":{"operation":"INSERT","newValue":"2018-03-24T13:56:45.000+05:30"},"contractTitle":{"operation":"INSERT","newValue":[{"contractTitleLineType":"lgl_ttl_ln_1","contractTitleLine":"Tiffany Riley"},{"contractTitleLineType":"lgl_ttl_ln_2","contractTitleLine":"Matthew Davies"}]},"taxIdentifier":{"operation":"INSERT","newValue":{"taxIdType":"EIN","taxId":"ZLCK91795330413525"}},"contractBranchCode":{"operation":"INSERT","newValue":"ACXMGBA5"},"contractCountry":{"operation":"INSERT","newValue":"Mexico"},"partyRelations":[{"partyIdentifier":{"operation":"INSERT","newValue":"9823462810"},"partyRelationshipType":{"operation":"INSERT","newValue":"F-N"},"partyRelationStartDateTime":{"operation":"INSERT","newValue":"2019-07-29T06:21:32.000+05:30"},"partyAddress":{"operation":"INSERT","newValue":{"addressLine1":"45229 Drake Route","addressLine2":"13306 Corey Point","addressCity":"Shanefort","addressPostalCode":"77163","addressCountry":"Canada","addressStartDate":"2019-02-26"}}}]}}
2+
{"eventHeader":{"eventIdentifier":"1c1f4df6-8d4f-4c52-9ba6-165babe48f67","eventType":"SBDL-Contract","majorSchemaVersion":1,"minorSchemaVersion":0,"eventDateTime":"2022-09-06T20:49:03+0530"},"keys":[{"keyField":"contractIdentifier","keyValue":"6982391061"}],"payload":{"contractIdentifier":{"operation":"INSERT","newValue":"6982391061"},"sourceSystemIdentifier":{"operation":"INSERT","newValue":"ADS"},"contactStartDateTime":{"operation":"INSERT","newValue":"2018-07-19T11:24:49.000+05:30"},"contractTitle":{"operation":"INSERT","newValue":[{"contractTitleLineType":"lgl_ttl_ln_1","contractTitleLine":"Garcia and Sons"},{"contractTitleLineType":"lgl_ttl_ln_2","contractTitleLine":"Taylor Guzman"}]},"taxIdentifier":{"operation":"INSERT","newValue":{"taxIdType":"SSP","taxId":"CADU39916151090321"}},"contractBranchCode":{"operation":"INSERT","newValue":"SHJFGBML"},"contractCountry":{"operation":"INSERT","newValue":"United States"},"partyRelations":[{"partyIdentifier":{"operation":"INSERT","newValue":"9823462811"},"partyRelationshipType":{"operation":"INSERT","newValue":"F-N"},"partyRelationStartDateTime":{"operation":"INSERT","newValue":"2018-08-31T05:27:22.000+05:30"},"partyAddress":{"operation":"INSERT","newValue":{"addressLine1":"361 Robinson Green","addressLine2":"3511 Rebecca Mission","addressCity":"North Tyler","addressPostalCode":"34118","addressCountry":"Canada","addressStartDate":"2018-01-28"}}}]}}
3+
{"eventHeader":{"eventIdentifier":"38d57237-a566-4ec4-a471-f659e9ce8437","eventType":"SBDL-Contract","majorSchemaVersion":1,"minorSchemaVersion":0,"eventDateTime":"2022-09-06T20:49:03+0530"},"keys":[{"keyField":"contractIdentifier","keyValue":"6982391067"}],"payload":{"contractIdentifier":{"operation":"INSERT","newValue":"6982391067"},"sourceSystemIdentifier":{"operation":"INSERT","newValue":"BDL"},"contactStartDateTime":{"operation":"INSERT","newValue":"2018-08-29T17:18:59.000+05:30"},"contractTitle":{"operation":"INSERT","newValue":[{"contractTitleLineType":"lgl_ttl_ln_1","contractTitleLine":"Acosta Inc"},{"contractTitleLineType":"lgl_ttl_ln_2","contractTitleLine":"David Walker"}]},"taxIdentifier":{"operation":"INSERT","newValue":{"taxIdType":"SSP","taxId":"UJLN20870916345792"}},"contractBranchCode":{"operation":"INSERT","newValue":"WZTEGBTG"},"contractCountry":{"operation":"INSERT","newValue":"Canada"},"partyRelations":[{"partyIdentifier":{"operation":"INSERT","newValue":"9823462817"},"partyRelationshipType":{"operation":"INSERT","newValue":"F-N"},"partyRelationStartDateTime":{"operation":"INSERT","newValue":"2018-05-16T09:53:04.000+05:30"},"partyAddress":{"operation":"INSERT","newValue":{"addressLine1":"251 Lee Tunnel","addressLine2":"09795 Tara Station Suite 264","addressCity":"New Michelleborough","addressPostalCode":"05505","addressCountry":"United States","addressStartDate":"2019-04-20"}}},{"partyIdentifier":{"operation":"INSERT","newValue":"9823462820"},"partyRelationshipType":{"operation":"INSERT","newValue":"F-S"},"partyRelationStartDateTime":{"operation":"INSERT","newValue":"2017-11-20T14:18:05.000+05:30"}},{"partyIdentifier":{"operation":"INSERT","newValue":"9823462821"},"partyRelationshipType":{"operation":"INSERT","newValue":"F-S"},"partyRelationStartDateTime":{"operation":"INSERT","newValue":"2018-07-19T18:56:57.000+05:30"}}]}}
4+
{"eventHeader":{"eventIdentifier":"38b24610-3bb9-4adc-8f8c-cfc3c24877f2","eventType":"SBDL-Contract","majorSchemaVersion":1,"minorSchemaVersion":0,"eventDateTime":"2022-09-06T20:49:03+0530"},"keys":[{"keyField":"contractIdentifier","keyValue":"6982391064"}],"payload":{"contractIdentifier":{"operation":"INSERT","newValue":"6982391064"},"sourceSystemIdentifier":{"operation":"INSERT","newValue":"ADS"},"contactStartDateTime":{"operation":"INSERT","newValue":"2018-03-28T15:47:43.000+05:30"},"contractTitle":{"operation":"INSERT","newValue":[{"contractTitleLineType":"lgl_ttl_ln_1","contractTitleLine":"Allen Group"}]},"taxIdentifier":{"operation":"INSERT","newValue":{"taxIdType":"CPR","taxId":"WJMX61093523376960"}},"contractBranchCode":{"operation":"INSERT","newValue":"OCCKGB65"},"contractCountry":{"operation":"INSERT","newValue":"Canada"},"partyRelations":[{"partyIdentifier":{"operation":"INSERT","newValue":"9823462814"},"partyRelationshipType":{"operation":"INSERT","newValue":"F-N"},"partyRelationStartDateTime":{"operation":"INSERT","newValue":"2019-06-06T14:18:12.000+05:30"},"partyAddress":{"operation":"INSERT","newValue":{"addressLine1":"5227 Wagner Pines","addressLine2":"189 Julie Throughway","addressCity":"West Amanda","addressPostalCode":"78962","addressCountry":"Canada","addressStartDate":"2018-07-11"}}}]}}
5+
{"eventHeader":{"eventIdentifier":"40a7cb7f-3996-47db-85ae-13d08d5998ea","eventType":"SBDL-Contract","majorSchemaVersion":1,"minorSchemaVersion":0,"eventDateTime":"2022-09-06T20:49:03+0530"},"keys":[{"keyField":"contractIdentifier","keyValue":"6982391065"}],"payload":{"contractIdentifier":{"operation":"INSERT","newValue":"6982391065"},"sourceSystemIdentifier":{"operation":"INSERT","newValue":"COR"},"contactStartDateTime":{"operation":"INSERT","newValue":"2018-06-30T22:33:19.000+05:30"},"contractTitle":{"operation":"INSERT","newValue":[{"contractTitleLineType":"lgl_ttl_ln_1","contractTitleLine":"Austin Miles"}]},"taxIdentifier":{"operation":"INSERT","newValue":{"taxIdType":"EIN","taxId":"TYAB75470638120665"}},"contractBranchCode":{"operation":"INSERT","newValue":"XCVKGB49"},"contractCountry":{"operation":"INSERT","newValue":"Canada"},"partyRelations":[{"partyIdentifier":{"operation":"INSERT","newValue":"9823462815"},"partyRelationshipType":{"operation":"INSERT","newValue":"F-N"},"partyRelationStartDateTime":{"operation":"INSERT","newValue":"2019-05-04T05:12:37.000+05:30"},"partyAddress":{"operation":"INSERT","newValue":{"addressLine1":"6993 Diane Alley","addressLine2":"8945 Trevor Greens","addressCity":"Kendrafurt","addressPostalCode":"50790","addressCountry":"United States","addressStartDate":"2017-10-08"}}}]}}
6+
{"eventHeader":{"eventIdentifier":"b96bc28e-e949-4f08-ae7d-e0787d2b02fc","eventType":"SBDL-Contract","majorSchemaVersion":1,"minorSchemaVersion":0,"eventDateTime":"2022-09-06T20:49:03+0530"},"keys":[{"keyField":"contractIdentifier","keyValue":"6982391066"}],"payload":{"contractIdentifier":{"operation":"INSERT","newValue":"6982391066"},"sourceSystemIdentifier":{"operation":"INSERT","newValue":"BDL"},"contactStartDateTime":{"operation":"INSERT","newValue":"2017-08-25T03:00:00.000+05:30"},"contractTitle":{"operation":"INSERT","newValue":[{"contractTitleLineType":"lgl_ttl_ln_1","contractTitleLine":"Miss Lisa Lee"}]},"taxIdentifier":{"operation":"INSERT","newValue":{"taxIdType":"SSP","taxId":"WFDP61047142851240"}},"contractBranchCode":{"operation":"INSERT","newValue":"CVYEGBJC"},"contractCountry":{"operation":"INSERT","newValue":"United States"},"partyRelations":[{"partyIdentifier":{"operation":"INSERT","newValue":"9823462816"},"partyRelationshipType":{"operation":"INSERT","newValue":"F-N"},"partyRelationStartDateTime":{"operation":"INSERT","newValue":"2019-05-15T10:39:29.000+05:30"},"partyAddress":{"operation":"INSERT","newValue":{"addressLine1":"23450 Timothy Divide","addressLine2":"125 Johnson Mountain Suite 701","addressCity":"Osbornetown","addressPostalCode":"04756","addressCountry":"Canada","addressStartDate":"2018-11-28"}}}]}}
7+
{"eventHeader":{"eventIdentifier":"2d6234a7-959e-4b89-837e-58f45d66d734","eventType":"SBDL-Contract","majorSchemaVersion":1,"minorSchemaVersion":0,"eventDateTime":"2022-09-06T20:49:03+0530"},"keys":[{"keyField":"contractIdentifier","keyValue":"6982391062"}],"payload":{"contractIdentifier":{"operation":"INSERT","newValue":"6982391062"},"sourceSystemIdentifier":{"operation":"INSERT","newValue":"CML"},"contactStartDateTime":{"operation":"INSERT","newValue":"2017-09-24T08:14:17.000+05:30"},"contractTitle":{"operation":"INSERT","newValue":[{"contractTitleLineType":"lgl_ttl_ln_1","contractTitleLine":"Theresa Mays"}]},"taxIdentifier":{"operation":"INSERT","newValue":{"taxIdType":"CPR","taxId":"WTUP76582369402245"}},"contractBranchCode":{"operation":"INSERT","newValue":"XCVKGB49"},"contractCountry":{"operation":"INSERT","newValue":"Canada"},"partyRelations":[{"partyIdentifier":{"operation":"INSERT","newValue":"9823462812"},"partyRelationshipType":{"operation":"INSERT","newValue":"F-N"},"partyRelationStartDateTime":{"operation":"INSERT","newValue":"2018-08-25T15:50:29.000+05:30"},"partyAddress":{"operation":"INSERT","newValue":{"addressLine1":"039 Daniel Mount","addressLine2":"8219 Hernandez Lodge Suite 875","addressCity":"Boltonborough","addressPostalCode":"71648","addressCountry":"Mexico","addressStartDate":"2018-12-07"}}}]}}
8+
{"eventHeader":{"eventIdentifier":"f752ac70-9992-44a5-b5af-579086578c2c","eventType":"SBDL-Contract","majorSchemaVersion":1,"minorSchemaVersion":0,"eventDateTime":"2022-09-06T20:49:03+0530"},"keys":[{"keyField":"contractIdentifier","keyValue":"6982391068"}],"payload":{"contractIdentifier":{"operation":"INSERT","newValue":"6982391068"},"sourceSystemIdentifier":{"operation":"INSERT","newValue":"BDL"},"contactStartDateTime":{"operation":"INSERT","newValue":"2018-05-11T15:24:57.000+05:30"},"contractTitle":{"operation":"INSERT","newValue":[{"contractTitleLineType":"lgl_ttl_ln_1","contractTitleLine":"Amanda Martin"}]},"taxIdentifier":{"operation":"INSERT","newValue":{"taxIdType":"EIN","taxId":"JDPX30428146546118"}},"contractBranchCode":{"operation":"INSERT","newValue":"ACXMGBA5"},"contractCountry":{"operation":"INSERT","newValue":"Mexico"},"partyRelations":[{"partyIdentifier":{"operation":"INSERT","newValue":"9823462818"},"partyRelationshipType":{"operation":"INSERT","newValue":"F-N"},"partyRelationStartDateTime":{"operation":"INSERT","newValue":"2017-11-27T01:20:12.000+05:30"},"partyAddress":{"operation":"INSERT","newValue":{"addressLine1":"7537 Clarke Club","addressLine2":"74089 Jerry Trail","addressCity":"Hunterville","addressPostalCode":"19596","addressCountry":"United States","addressStartDate":"2018-07-17"}}}]}}

Diff for: ‎SBDL - Starter/test_pytest_sbdl.py

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import pytest
2+
3+
from lib.Utils import get_spark_session
4+
5+
6+
@pytest.fixture(scope='session')
7+
def spark():
8+
return get_spark_session("LOCAL")
9+
10+
11+
def test_blank_test(spark):
12+
print(spark.version)
13+
assert spark.version == "3.3.0"
14+
15+

Diff for: ‎SBDL/.gitignore

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Default ignored files
2+
/shelf/
3+
/workspace.xml
4+
.idea/*
5+
lib/__pycache__/*

0 commit comments

Comments
 (0)