-
Notifications
You must be signed in to change notification settings - Fork 12
/
Copy pathMakefile
139 lines (110 loc) · 6.55 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
install:
./gradlew compileJava installDist shadowJar
test:
./gradlew test -x spotbugsMain -x spotbugsTest -x spotbugsTestFixtures
build:
./gradlew build
docker build . -t hoptimator
docker build hoptimator-flink-runner -f hoptimator-flink-runner/Dockerfile-flink-runner -t hoptimator-flink-runner
docker build hoptimator-flink-runner -f hoptimator-flink-runner/Dockerfile-flink-operator -t hoptimator-flink-operator
bounce: build undeploy deploy deploy-samples deploy-config deploy-demo
clean:
./gradlew clean
deploy-config:
kubectl apply -f ./deploy/config/hoptimator-configmap.yaml
undeploy-config:
kubectl delete configmap hoptimator-configmap || echo "skipping"
deploy: deploy-config
kubectl apply -f ./hoptimator-k8s/src/main/resources/
kubectl apply -f ./deploy
kubectl apply -f ./deploy/dev/rbac.yaml
undeploy: undeploy-config
kubectl delete -f ./deploy/dev/rbac.yaml || echo "skipping"
kubectl delete -f ./deploy || echo "skipping"
kubectl delete -f ./hoptimator-k8s/src/main/resources/ || echo "skipping"
quickstart: build deploy
deploy-demo: deploy
kubectl apply -f ./deploy/samples/demodb.yaml
undeploy-demo: undeploy
kubectl delete -f ./deploy/samples/demodb.yaml
deploy-samples: deploy
kubectl wait --for=condition=Established=True \
crds/subscriptions.hoptimator.linkedin.com \
crds/kafkatopics.hoptimator.linkedin.com \
crds/sqljobs.hoptimator.linkedin.com
kubectl apply -f ./deploy/samples
undeploy-samples: undeploy
kubectl delete -f ./deploy/samples || echo "skipping"
deploy-flink: deploy
kubectl create -f https://github.com/jetstack/cert-manager/releases/download/v1.8.2/cert-manager.yaml || echo "skipping"
helm repo add flink-operator-repo https://downloads.apache.org/flink/flink-kubernetes-operator-1.9.0/
helm upgrade --install --atomic --set webhook.create=false,image.pullPolicy=Never,image.repository=docker.io/library/hoptimator-flink-operator,image.tag=latest flink-kubernetes-operator flink-operator-repo/flink-kubernetes-operator
kubectl apply -f deploy/samples/flinkDeployment.yaml
kubectl apply -f deploy/samples/flinkSessionJob.yaml
docker compose -f ./deploy/docker/flink/docker-compose-sql-gateway.yaml up -d --wait
undeploy-flink:
docker compose -f ./deploy/docker/flink/docker-compose-sql-gateway.yaml down
kubectl delete flinksessionjobs.flink.apache.org --all || echo "skipping"
kubectl delete flinkdeployments.flink.apache.org --all || echo "skipping"
kubectl delete crd flinksessionjobs.flink.apache.org || echo "skipping"
kubectl delete crd flinkdeployments.flink.apache.org || echo "skipping"
helm uninstall flink-kubernetes-operator || echo "skipping"
helm repo remove flink-operator-repo || echo "skipping"
kubectl delete -f https://github.com/jetstack/cert-manager/releases/download/v1.8.2/cert-manager.yaml || echo "skipping"
deploy-kafka: deploy deploy-flink
kubectl create namespace kafka || echo "skipping"
kubectl apply -f "https://strimzi.io/install/latest?namespace=kafka" -n kafka
kubectl wait --for=condition=Established=True crds/kafkas.kafka.strimzi.io
kubectl apply -f ./deploy/dev
kubectl apply -f ./deploy/samples/demodb.yaml
kubectl apply -f ./deploy/samples/kafkadb.yaml
undeploy-kafka:
kubectl delete kafkatopic.kafka.strimzi.io -n kafka --all || echo "skipping"
kubectl delete strimzi -n kafka --all || echo "skipping"
kubectl delete pvc -l strimzi.io/name=one-kafka -n kafka || echo "skipping"
kubectl delete -f "https://strimzi.io/install/latest?namespace=kafka" -n kafka || echo "skipping"
kubectl delete -f ./deploy/samples/kafkadb.yaml || echo "skipping"
kubectl delete -f ./deploy/samples/demodb.yaml || echo "skipping"
kubectl delete -f ./deploy/dev || echo "skipping"
kubectl delete namespace kafka || echo "skipping"
# Deploys Venice cluster in docker and creates two stores in Venice. Stores are not managed via K8s for now.
deploy-venice: deploy deploy-flink
docker compose -f ./deploy/docker/venice/docker-compose-single-dc-setup.yaml up -d --wait
docker exec venice-client ./create-store.sh http://venice-controller:5555 venice-cluster0 test-store schemas/keySchema.avsc schemas/valueSchema.avsc
docker exec venice-client ./create-store.sh http://venice-controller:5555 venice-cluster0 test-store-1 schemas/keySchema.avsc schemas/valueSchema.avsc
kubectl apply -f ./deploy/samples/venicedb.yaml
undeploy-venice:
kubectl delete -f ./deploy/samples/venicedb.yaml || echo "skipping"
docker compose -f ./deploy/docker/venice/docker-compose-single-dc-setup.yaml down
deploy-dev-environment: deploy deploy-flink deploy-kafka deploy-venice
undeploy-dev-environment: undeploy-venice undeploy-kafka undeploy-flink undeploy
# Integration test setup intended to be run locally
integration-tests: deploy-dev-environment deploy-samples
kubectl wait kafka.kafka.strimzi.io/one --for=condition=Ready --timeout=10m -n kafka
kubectl wait kafkatopic.kafka.strimzi.io/existing-topic-1 --for=condition=Ready --timeout=10m -n kafka
kubectl wait kafkatopic.kafka.strimzi.io/existing-topic-2 --for=condition=Ready --timeout=10m -n kafka
kubectl port-forward -n kafka svc/one-kafka-external-bootstrap 9092 & echo $$! > port-forward.pid
./gradlew intTest || kill `cat port-forward.pid`
kill `cat port-forward.pid`
# kind cluster used in github workflow needs to have different routing set up, avoiding the need to forward kafka ports
integration-tests-kind: deploy-dev-environment deploy-samples
kubectl wait kafka.kafka.strimzi.io/one --for=condition=Ready --timeout=10m -n kafka
kubectl wait kafkatopic.kafka.strimzi.io/existing-topic-1 --for=condition=Ready --timeout=10m -n kafka
kubectl wait kafkatopic.kafka.strimzi.io/existing-topic-2 --for=condition=Ready --timeout=10m -n kafka
./gradlew intTest
generate-models:
./generate-models.sh
./hoptimator-models/generate-models.sh # <-- marked for deletion
release:
test -n "$(VERSION)" # MISSING ARG: $$VERSION
./gradlew publish
build-zeppelin:
docker build -t hoptimator-zeppelin -t hoptimator-zeppelin:0.11.2 -f ./deploy/docker/zeppelin/Dockerfile-zeppelin .
# attaches to terminal (not run as daemon)
run-zeppelin:
docker run --rm -p 8080:8080 \
--volume=${HOME}/.kube/config:/opt/zeppelin/.kube/config \
--add-host=docker-for-desktop:host-gateway \
--name hoptimator-zeppelin \
hoptimator-zeppelin
.PHONY: install test build bounce clean quickstart deploy-config undeploy-config deploy undeploy deploy-demo undeploy-demo deploy-samples undeploy-samples deploy-flink undeploy-flink deploy-kafka undeploy-kafka deploy-venice undeploy-venice build-zeppelin run-zeppelin integration-tests integration-tests-kind deploy-dev-environment undeploy-dev-environment generate-models release