-
Notifications
You must be signed in to change notification settings - Fork 0
/
README.txt
2111 lines (1554 loc) · 73.6 KB
/
README.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#################################################################
#For Master Install Postgres
yum install https://download.postgresql.org/pub/repos/yum/10/redhat/rhel-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm -y
sleep 2
yum install postgresql10 postgresql10-server postgresql10-contrib postgresql10-libs -y
sleep 1
/usr/pgsql-10/bin/postgresql-10-setup initdb
sleep 2
systemctl enable postgresql-10.service && systemctl start postgresql-10.service && systemctl status postgresql-10.service
su – postgres
psql
sudo /opt/cloudera/cm/schema/scm_prepare_database.sh postgresql fly postgres
create database bigdata;
create database hive;
create database amon;
create database rman;
create database oozie_oozie_server;
create database hue;
create database sentry;
create database nav;
create database navms;
create database clouderascm;
create user hive with encrypted password 'markv109';
create user amon with encrypted password 'markv109';
create user rman with encrypted password 'markv109';
create user oozie_oozie_server with encrypted password 'markv109';
create user hue with encrypted password 'markv109';
create user sentry with encrypted password 'markv109';
create user nav with encrypted password 'markv109';
create user navms with encrypted password 'markv109';
create user clouderascm with encrypted password 'markv109';
create user noknok with encrypted password 'markv109';
grant all privileges on database bigdata to noknok;
grant all privileges on database hive to noknok;
grant all privileges on database amon to noknok;
grant all privileges on database rman to noknok;
grant all privileges on database oozie_oozie_server to noknok;
grant all privileges on database hue to noknok;
grant all privileges on database sentry to noknok;
grant all privileges on database nav to noknok;
grant all privileges on database navms to noknok;
grant all privileges on database clouderascm to noknok;
#################################################################
#For Master M
#MAKE SURE YOU HAVE SET THE TIMEZONE TO PHILIPPINES
#DOUBLE CHECK IF THE FIREWALLD is DISABLE
STEP 1
#!/bin/bash
echo never > /sys/kernel/mm/transparent_hugepage/enabled
sleep 1
echo never > /sys/kernel/mm/transparent_hugepage/defrag
sleep 1
sysctl vm.swappiness=1
sleep 1
iptables-save > ~/firewall.rules
sleep 1
systemctl stop firewalld
sleep 1
systemctl disable firewalld
sed -i "/^SELINUX=enforcing*/c\SELINUX=disabled" /etc/selinux/config
#dnf install -y https://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/e/epel-release-7-13.noarch.rpm
init 6
clear
STEP 2
#!/bin/bash
yum -y install ntp
sleep 1
systemctl start ntpd
sleep 1
systemctl enable ntpd
sleep 1
systemctl status ntpd
sleep 1
systemctl status chronyd
slee5 1
systemctl disable chronyd
hostname $(hostname -I)
sleep 1
yum -y install java-1.8.0-openjdk-devel
sleep 1
systemctl disable autofs
sleep 1
systemctl is-enabled autofs
cd /boot/grub2
stat /boot/grub2/grub.cfg
chmod og-rwx /boot/grub2/grub.cfg
echo "copy and execute"
echo "grub2-mkconfig > /boot/grub2/grub.cfg"
echo "grub2-mkconfig > /boot/grub2/grub.cfg"
grub2-mkpasswd-pbkdf2
Step 3 SETUP YOUR IP IN /ETC/HOSTS
#!/bin/bash
yum remove prelink
cd /home
sleep 1
yum -y install httpd
sleep 1
systemctl start httpd
sleep 1
systemctl enable httpd
sleep 1
systemctl status httpd
#touch hosts.txt
#echo -n "" > hosts.txt
#echo -n "" > /etc/hosts
#echo "10.128.0.52 instance-1.trisha.com master1" >> hosts.txt
#echo "10.128.0.53 instance-2.trisha.com worker1" >> hosts.txt
#echo "10.142.0.18 instance-3.trisha.com worker2" >> hosts.txt
#echo "10.128.0.54 instance-4.trisha.com worker3" >> hosts.txt
#echo "10.128.0.55 instance-5.trisha.com worker4" >> hosts.txt
sleep 1
#touch hosts.txt
#echo -n "" > hosts.txt
#echo -n "" > /etc/hosts
#echo "35.238.126.251 master1.trisha.com ma1" >> hosts.txt
#echo "35.232.230.24 worker1.trisha.com wm1" >> hosts.txt
#echo "35.202.100.194 worker2.trisha.com wm2" >> hosts.txt
#echo "34.121.21.128 worker3.trisha.com wm3" >> hosts.txt
#echo "34.70.36.0 worker4.trisha.com wm4" >> hosts.txt
#echo "34.122.245.56 worker5.trisha.com wm5" >> hosts.txt
#cat hosts.txt >> /etc/hosts
#systemctl restart systemd-hostnamed
#systemctl restart network
sed -i "/^PasswordAuthentication no*/c\PasswordAuthentication yes" /etc/ssh/sshd_config
sed -i "/^PermitRootLogin no*/c\PermitRootLogin yes" /etc/ssh/sshd_config
systemctl restart sshd
sed -i "/^## Allow root to run any commands anywhere*/c\noknok ALL=(ALL) ALL" /etc/sudoers
echo "create username noknok,sudo su noknok, then run the ssh-keygen"
echo "ssh-copy-id -i ~/.ssh/id_rsa.pub root@192.168.2.x"
Step 4
#!/bin/bash
cd /home
sleep 1
mkdir -p /var/www/html/cloudera-repos/
mkdir -p /var/www/html/cloudera-repos/cm6
cd /var/www/html/cloudera-repos/cm6
yum install wget -y
wget http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cm6/redhat/7/x86_64/cm/6.2.0/RPMS/x86_64/cloudera-manager-server-db-2-6.2.0-968826.el7.x86_64.rpm \
wget http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cm6/redhat/7/x86_64/cm/6.2.0/RPMS/x86_64/cloudera-manager-server-6.2.0-968826.el7.x86_64.rpm \
wget http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cm6/redhat/7/x86_64/cm/6.2.0/RPMS/x86_64/cloudera-manager-daemons-6.2.0-968826.el7.x86_64.rpm \
wget http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cm6/redhat/7/x86_64/cm/6.2.0/RPMS/x86_64/cloudera-manager-agent-6.2.0-968826.el7.x86_64.rpm \
wget http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cm6/redhat/7/x86_64/cm/6.2.0/RPMS/x86_64/enterprise-debuginfo-6.2.0-968826.el7.x86_64.rpm \
wget http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cm6/redhat/7/x86_64/cm/6.2.0/RPMS/x86_64/oracle-j2sdk1.8-1.8.0%2Bupdate181-1.x86_64.rpm \
wget https://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/e/epel-release-7-13.noarch.rpm
sleep 1
yum install -y cloudera-manager-daemons-6.2.0-968826.el7.x86_64.rpm cloudera-manager-server-6.2.0-968826.el7.x86_64.rpm cloudera-manager-agent-6.2.0-968826.el7.x86_64.rpm cloudera-manager-server-db-2-6.2.0-968826.el7.x86_64.rpm oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm
#yum clean all
#yum update -y
#cd /etc/yum.repos.d/
#rm -r -f cloudera-manager-repo
#echo -n "" > /etc/yum.repos.d/cloudera-manager.repo
#echo -n "" > repo2.txt
#echo "[cloudera-repo] " >> repo2.txt
#echo "name=cloudera-manager " >> repo2.txt
#echo "baseurl=http://35.237.185.141/cloudera-repos/cdh/ " >> repo2.txt
#echo "enabled=1 " >> repo2.txt
#echo "gpgcheck=0 " >> repo2.txt
#touch -a cloudera-manager.repo
#cat repo2.txt >> cloudera-manager.repo
cd /etc/yum.repos.d/
rm -r -f cloudera-manager-repo
echo -n "" > /etc/yum.repos.d/cloudera-manager.repo
echo -n "" > repo2.txt
echo "[cloudera-manager] " >> repo2.txt
echo "# Packages for Cloudera Manager, Version 6, on RedHat or CentOS 7 x86_64 " >> repo2.txt
echo "name=Cloudera Manager " >> repo2.txt
echo "#baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5/ " >> repo2.txt
echo "baseurl=http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cm6/redhat/7/x86_64/cm/6.2.0/ " >> repo2.txt
echo "#gpgkey=http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cm5/redhat/7/x86_64/cm/RPM-GPG-KEY-cloudera " >> repo2.txt
echo "#gpgkey=http://39.98.155.100/cloudera-repos/cm6/6.2.1/redhat7/yum/RPM-GPG-KEY-cloudera " >> repo2.txt
echo "enable = 1 " >> repo2.txt
echo "gpgcheck = 1 " >> repo2.txt
touch -a cloudera-manager.repo
cat repo2.txt >> cloudera-manager.repo
#listening_ip=listening_ip=34.70.36.0
#listening_hostname=listening_hostname=34.70.36.0
#sed -i "/^server_host=localhost*/c$server_host" /etc/cloudera-scm-agent/config.ini
#sed -i "/^# listening_ip=*/c$listening_ip" /etc/cloudera-scm-agent/config.ini
#sed -i "/^# listening_hostname=*/c$listening_hostname" /etc/cloudera-scm-agent/config.ini
touch /etc/cloudera-scm-server/db.properties
systemctl start cloudera-scm-agent && systemctl enable cloudera-scm-agent && systemctl status cloudera-scm-agent
systemctl start cloudera-scm-server && systemctl enable cloudera-scm-server && systemctl status cloudera-scm-server
sleep 3
systemctl start cloudera-scm-server-db && systemctl enable cloudera-scm-server-db && systemctl status cloudera-scm-server-db
rm -rf /rms/var/lib/oozie/*
rm -f -r /var/lib/cloudera-scm-agent/cm_guid
sleep 1
mkdir /var/www/html/cloudera-repos/cdh
cd /var/www/html/cloudera-repos/cdh
#wget http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cdh6/parcels/latest/CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel.sha256 \
#wget http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cdh6/parcels/latest/CDH-6.2.0-1.cdh6.2.0.p0.967373-el7.parcel \
#wget http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cdh6/parcels/latest/manifest.json
cd /home
rm -rf /rms/var/lib/oozie/*
rm -f -r /var/lib/cloudera-scm-agent/cm_guid
netstat -tulpn |grep LISTEN
echo "CHANGE THE SERVICE IP ADDRESS IN AGEN CONFIG FILES"
echo " watch 'tail /var/log/cloudera-scm-server/cloudera-scm-server.log' "
#Run the command if the admin page giving you error
touch /etc/cloudera-scm-server/db.mgmt.properties
touch /etc/cloudera-scm-server/db.properties
rm -rf /etc/cloudera-scm-server/db.properties.*
rm -rf /rms/var/lib/oozie/*
rm -f -r /var/lib/cloudera-scm-agent/cm_guid
init 6
to access
http://localhost:7180
#CDH Parcel
http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cdh6/parcels/6.2.0/
#IN AWS Server Ports must be open
7180, 7182, 80, 443, 22
#################################################
Troubleshooting in Cloudera
#TO have healthy host
rm /var/lib/cloudera-scm-agent/cm_guid
#Failing to install oozie
rm -rf /rms/var/lib/oozie/*
##################################################
additional execution for workers #
input the ip address in for hosts and server_host#
##################################################
rm /var/lib/cloudera-scm-agent/cm_guid
rm -rf /rms/var/lib/oozie/*
hostname $(hostname -I)
server_host=server_host=localhost
sed -i "/^server_host=localhost*/c$server_host" /etc/cloudera-scm-agent/config.ini
systemctl restart cloudera-scm-agent && systemctl status cloudera-scm-agent
systemctl stop cloudera-scm-server && systemctl stop cloudera-scm-server-db
sleep 1
rm -rf /rms/var/lib/oozie/*
rm -f -r /var/lib/cloudera-scm-agent/cm_guid
hostname $(hostname -I)
touch hosts.txt
echo -n "" > hosts.txt
echo -n "" > /etc/hosts
echo "192.168.2.112 master1.trisha.com ma1" >> hosts.txt
echo "10.128.0.6 worker1.trisha.com wm1" >> hosts.txt
echo "10.128.0.7 worker2.trisha.com wm2" >> hosts.txt
echo "10.128.0.8 worker3.trisha.com wm3" >> hosts.txt
echo "34.70.36.0 worker4.trisha.com wm4" >> hosts.txt
echo "34.122.245.56 worker5.trisha.com wm5" >> hosts.txt
echo "127.0.0.1 localhost" >> hosts.txt
cat hosts.txt >> /etc/hosts
systemctl restart systemd-hostnamed
systemctl restart network
#For localhost installation
sleep 1
rm -rf /rms/var/lib/oozie/*
rm -f -r /var/lib/cloudera-scm-agent/cm_guid
hostname $(hostname -I)
touch hosts.txt
echo -n "" > hosts.txt
echo -n "" > /etc/hosts
echo "192.168.2.28 localhost" >> hosts.txt
echo "127.0.0.1 master1.trisha.com" >> hosts.txt
cat hosts.txt >> /etc/hosts
systemctl restart systemd-hostnamed
systemctl restart network
##################################################
additional execution for workers #
input the ip address in for hosts and server_host#
##################################################
hostname $(hostname -I)
systemctl stop cloudera-scm-server
systemctl stop cloudera-scm-server-db
systemctl disable cloudera-scm-server
systemctl disable cloudera-scm-server-db
server_host=server_host=localhost
sed -i "/^server_host=localhost*/c$server_host" /etc/cloudera-scm-agent/config.ini
systemctl restart cloudera-scm-agent && systemctl status cloudera-scm-agent
sleep 1
rm -rf /rms/var/lib/oozie/*
rm -f -r /var/lib/cloudera-scm-agent/cm_guid
touch hosts.txt
echo -n "" > hosts.txt
echo -n "" > /etc/hosts
echo "192.168.2.248 master1.trisha.com ma1" >> hosts.txt
echo "192.168.2.42 worker1.trisha.com wm1" >> hosts.txt
echo "35.239.128.56 worker2.trisha.com wm2" >> hosts.txt
echo "34.121.21.128 worker3.trisha.com wm3" >> hosts.txt
echo "34.70.36.0 worker4.trisha.com wm4" >> hosts.txt
echo "34.122.245.56 worker5.trisha.com wm5" >> hosts.txt
echo "127.0.0.1 localhost" >> hosts.txt
cat hosts.txt >> /etc/hosts
systemctl restart systemd-hostnamed
systemctl restart network
shutdown now
to check log files
watch 'head -n 2 /var/log/cloudera-scm-server/cloudera-scm-server.log;'
watch 'tail /var/log/cloudera-scm-server/cloudera-scm-server.log;'
#TO make it work in 1 localhost pc, 2 vm machines.
master node must be live ip
worker node must be inside ip that connects to live ip.
worker node listening ip/host must set inside ip
########################################################################
########################################################################
########################################################################
#netstat -tulpn | grep LISTEN
#REMINDER:
To make sure the agents are manage to the server, it is a MUST to set the server_hostname=10.x.x.x of server in /etc/cloudera-scm-agent/config.ini
no other necessary configuration to other files.
#BEFORE RUNNING THE SCRIPT, IT IS REQUIRED TO SECURE THE VM
useradd -m <new_user>
passwd <new_user>
mkdir tmp #directo for rsakeys
cd tmp/
ssh-keygen
#enter the folder /home/<username>/tmp/<filename_of_keys>, enter then enter
ssh-copy-id -i <folder/file_key> username@ip
[cloudera-cdh]
# Packages for Cloudera Manager, Version 5, on RedHat or CentOS 6 x86_64
name=Cloudera Manager
#baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5/
baseurl=http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cm6/redhat/7/x86_64/cm/6.2.0/
gpgkey=http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cm5/redhat/7/x86_64/cm/RPM-GPG-KEY-cloudera
gpgcheck = 1
[cloudera-manager]
# Packages for Cloudera Manager, Version 5, on RedHat or CentOS 6 x86_64
name=Cloudera Manager
#baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5/
baseurl=http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cm6/redhat/7/x86_64/cm/6.2.0/
#gpgkey=http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cm5/redhat/7/x86_64/cm/RPM-GPG-KEY-cloudera
gpgkey=http://39.98.155.100/cloudera-repos/cm6/6.2.1/redhat7/yum/RPM-GPG-KEY-cloudera
gpgcheck = 1
#How to install Hadoop
https://github.com/bright60/hadoop_install
# ALLKEYS.asc
https://mirror.one.com/cloudera/cm6/6.2.0/allkeys.asc
http://39.98.155.100/cloudera-repos/cm6/6.2.1/
#Cloudera RPM-GPN-Key
http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cm6/redhat/7/x86_64/cm/RPM-GPG-KEY-cloudera
http://39.98.155.100/cloudera-repos/cm6/6.2.1/redhat7/yum/RPM-GPG-KEY-cloudera
http://39.98.155.100/cloudera-repos/cm6/6.2.1/redhat7/yum/RPM-GPG-KEY-cloudera
#CDH Parcel
#http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cdh6/parcels/6.2.0/
#CM6 link
#http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cm6/redhat/7/x86_64/cm/6.2.0/RPMS/x86_64/
#http://39.98.155.100/cloudera-repos/cm6/6.2.1/redhat7/yum/
#REMIND to check the service-hostname in the /etc/cloudera-scm-agent/config it must be lastest master ip
#To adduser/change noknok to no password login to other server
adduser -m noknok
passwd noknok
vi /etc/sudoers
root ALL=(ALL) ALL
noknok ALL=(ALL) ALL
sed -i "/^## Allow root to run any commands anywhere*/c\noknok ALL=(ALL) ALL" /etc/sudoers
sed -i "/^PasswordAuthentication no*/c\PasswordAuthentication yes" /etc/ssh/sshd_config
systemctl restart sshd
sudo su noknok
ssh-keygen
ssh-copy-id -i ~/.ssh/id_rsa.pub noknok@ip
#To secure grub boot, should run in all master and worker servers.
cd /boot/grub2
stat /boot/grub2/grub.cfg
chmod og-rwx /boot/grub2/grub.cfg
grub2-mkpasswd-pbkdf2
grub2-mkconfig > /boot/grub2/grub.cfg
#how to install mysql server follow the link
https://docs.cloudera.com/documentation/enterprise/6/6.3/topics/cm_ig_mysql.html#cmig_topic_5_5_2
#Connecting mysql to jdbc, important in setting up mysql as database for production follow the link
#input the suggestion configuration file in /etc/my.cnf
https://docs.cloudera.com/documentation/enterprise/6/6.3/topics/prepare_cm_database.html
https://docs.cloudera.com/documentation/enterprise/6/6.3/topics/cm_ig_mysql.html#cmig_topic_5_5
#Creating SSH to login from windows into linux server without password using putty-ssh
https://www.howtoforge.com/how-to-configure-ssh-keys-authentication-with-putty-and-linux-server-in-5-quick-steps
# Steps how to install Cloudera in CeontOS
https://www.tecmint.com/best-practices-for-deploying-hadoop-server-on-centos/
#Generate RootCACert.pem RootCAKey.pem follow this instruction
https://www.ibm.com/docs/en/runbook-automation?topic=certificate-generate-root-ca-key
#To install TSL/SSL in Cloudera for security messures follow this instruction
https://docs.cloudera.com/documentation/enterprise/6/6.2/topics/sg_add_root_ca_explicit_trust.html
create this directory /opt/cloudera/security/pki/
download install oracle-jdk.18
wget http://ro-bucharest-repo.bigstepcloud.com/cloudera-repos/cm6/redhat/7/x86_64/cm/6.2.0/RPMS/x86_64/oracle-j2sdk1.8-1.8.0%2Bupdate181-1.x86_64.rpm
#How to secure Cloudera
openssl genrsa -out rootCAKey.pem 2048
openssl req -x509 -sha256 -new -nodes -key rootCAKey.pem -days 3650 -out rootCACert.pem
openssl x509 -in rootCACert.pem -text
mkdir /opt
mkdir /opt/cloudera/security/
mkdir /opt/cloudera/security/pki
mkdir /jre
mkdir /jre/lib/
mkdir /jre/lib/security/
cp root* /opt/cloudera/security/pki/
sudo keytool -importcert -alias rootca -keystore $JAVA_HOME/jre/lib/security/jssecacerts \
-file /opt/cloudera/security/pki/rootCACert.pem -storepass markv109
sudo cat /opt/cloudera/security/pki/rootCACert.pem >> \
/opt/cloudera/security/pki/$(hostname -f)-server.cert.pem
#How to download/install/configure mysql
yum install mysql-server
systemctl enable mysqld
systemctl start mysqld
/usr/bin/mysql_secure_installation
wget http://repo.mysql.com/mysql-community-release-el7-5.noarch.rpm
sudo rpm -ivh mysql-community-release-el7-5.noarch.rpm
sudo yum update -y
sudo yum install -y mysql-server
sudo systemctl start mysqld
/usr/bin/mysql_secure_installation
#Create databases for standard for service
create database rmon DEFAULT CHARACTER SET utf8;
create database hive DEFAULT CHARACTER SET utf8;
create database sentry DEFAULT CHARACTER SET utf8;
create database nav DEFAULT CHARACTER SET utf8;
create database navms DEFAULT CHARACTER SET utf8;
create database hue DEFAULT CHARACTER SET utf8;
create database clouderascm DEFAULT CHARACTER SET utf8;
#Create a database users and passwords.
grant all on bigdata.* TO 'rmon'@'%' IDENTIFIED BY 'markv109';
grant all on rmon.* TO 'rmon'@'%' IDENTIFIED BY 'markv109';
grant all on hive.* TO 'hive'@'%' IDENTIFIED BY 'markv109';
grant all on sentry.* TO 'sentry'@'%' IDENTIFIED BY 'markv109';
grant all on nav.* TO 'nav'@'%' IDENTIFIED BY 'markv109';
grant all on navms.* TO 'navms'@'%' IDENTIFIED BY 'markv109';
grant all on hue.* TO 'hue'@'%' IDENTIFIED BY 'markv109';
grant all on clouderascm.* TO 'clouderascm'@'%' IDENTIFIED BY 'markv109';
#create database oozie default character set utf8;
grant all privileges on oozie.* to 'oozie'@'localhost' identified by 'markv109';
grant all privileges on oozie.* to 'oozie'@'%' identified by 'markv109';
Ports need to be in in fre
7180,9864,7183,7187,7183,8020,50010,10110,10111,7182,7432,7190, 7191,9000,7184,7185, 8084, 10101, 8086, 9997, 9996, 8087, 9999, 9998, 8091, 9091, 9995, 9994, 5678, 8083, 7186, 8089
###############################################################################################
##################################################################################################################################
RHEL howto's
#EPEL For RHEL
dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
How to enable systemd on WSL2: Ubuntu 20 and CentOS 8
https://www.javaer101.com/en/article/11055970.html
Installating GNOME for RHEL
yum group install GNOME base-x -y
subscription-manager repos --enable=codeready-builder-for-rhel-8-x86_64-rpms
yum install xorg-x11-apps -y
https://packages.microsoft.com/config/rhel/8/
https://packages.microsoft.com/config/centos/8/
#How to install VNC Server
https://www.tecmint.com/install-vnc-server-on-rhel-8/
#To run the guiG
systemctl set-default graphical.target
rpm -i https://packages.microsoft.com/config/rhel/8/packages-microsoft-prod.rpm
wsl -d genie -s #running the RHWSL sysmtemd on #drawbacks are you cannot update
How to Fix “Failed to set locale, defaulting to C.UTF-8”
https://www.tecmint.com/fix-failed-to-set-locale-defaulting-to-c-utf-8-in-centos/
dnf install @postgresql -y
dnf install langpacks-en glibc-all-langpacks -y
dnf install glibc-langpack-en
#If registration is getting error
# subscription-manager refresh
# subscription-manager attach --auto
# subscription-manager status
# subscription-manager remove --all
# subscription-manager unregister
# subscription-manager clean
#Try again
# subscription-manager register
# subscription-manager attach --auto
#EPEL-Release for RHEL
yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
#VNC Files download ang install all, with --skip-broken paramater
https://homes.esat.kuleuven.be/~rtheys/vnc/
#how to install VNC in IBM
https://www.ibm.com/support/pages/how-configure-vnc-server-red-hat-enterprise-linux-8
#Download the old TigerVNC from the link
wget https://homes.esat.kuleuven.be/~rtheys/vnc/tigervnc-1.11.0-5.el8.esat~esat1.src.rpm \
wget https://homes.esat.kuleuven.be/~rtheys/vnc/tigervnc-1.11.0-5.el8.esat~esat1.x86_64.rpm \
wget https://homes.esat.kuleuven.be/~rtheys/vnc/tigervnc-icons-1.11.0-5.el8.esat~esat1.noarch.rpm \
wget https://homes.esat.kuleuven.be/~rtheys/vnc/tigervnc-license-1.11.0-5.el8.esat~esat1.noarch.rpm \
wget https://homes.esat.kuleuven.be/~rtheys/vnc/tigervnc-selinux-1.11.0-5.el8.esat~esat1.noarch.rpm \
wget https://homes.esat.kuleuven.be/~rtheys/vnc/tigervnc-server-1.11.0-5.el8.esat~esat1.x86_64.rpm \
wget https://homes.esat.kuleuven.be/~rtheys/vnc/tigervnc-server-minimal-1.11.0-5.el8.esat~esat1.x86_64.rpm \
wget https://homes.esat.kuleuven.be/~rtheys/vnc/tigervnc-server-module-1.11.0-5.el8.esat~esat1.x86_64.rpm
#Step by step to install VNC working in working in RHWSL8
yum install -y \
tigervnc-1.11.0-5.el8.esat~esat1.src.rpm \
tigervnc-1.11.0-5.el8.esat~esat1.x86_64.rpm \
tigervnc-icons-1.11.0-5.el8.esat~esat1.noarch.rpm \
tigervnc-license-1.11.0-5.el8.esat~esat1.noarch.rpm \
tigervnc-selinux-1.11.0-5.el8.esat~esat1.noarch.rpm \
tigervnc-server-1.11.0-5.el8.esat~esat1.x86_64.rpm \
tigervnc-server-minimal-1.11.0-5.el8.esat~esat1.x86_64.rpm \
tigervnc-server-module-1.11.0-5.el8.esat~esat1.x86_64.rpm \
--skip-broken
#Creat vncuser
su - noknok
vncpasswd
#Crete a service connection for VNC for OLD Style
cp /usr/lib/systemd/system/vncserver\@.service /etc/systemd/system/vncserver\@:1.service \
cp /usr/lib/systemd/system/vncserver\@.service /etc/systemd/system/vncserver\@:2.service \
cp /usr/lib/systemd/system/vncserver\@.service /etc/systemd/system/vncserver\@:3.service
vi /etc/systemd/system/vncserver@:3.service
#For latest VNC server
cp /usr/lib/systemd/system/vncserver\@.service /etc/systemd/system/vncserver\@.service
vi /etc/systemd/system/vncserver@.service
systemctl daemon-reload
systemctl start vncserver@:1
systemctl start vncserver@:2
systemctl start vncserver@:3
#Copy this configuration
[Service]
Type=forking
ExecStart=/usr/sbin/runuser -l noknok -c "/usr/bin/vncserver %i -geometry 1366x768"
PIDFile=/home/noknok/.vnc/%H%i.pid
#ExecStart=/usr/libexec/vncsession-start %i
#PIDFile=/run/vncsession-%i.pid
SELinuxContext=system_u:system_r:vnc_session_t:s0
#Restart daemon, so the configuration will take effect
systemctl daemon-reload
#Create start-up in the service
systemctl enable vncserver@:3.service
systemctl start vncserver@:3.service
#Before install nomachine
systemctl unmask systemd-logind
#No Machine EnableUserDB and EnablePasswordDB to 1 then
vi /usr/NX/etc/server.cfg
sudo /etc/NX/nxserver --useradd mark --system
sudo /etc/NX/nxserver --passwd <username>
#Firewalld
/etc/firewalld/firewalld.conf
firewall-cmd --zone=public --permanent --add-service=http
firewall-cmd --zone=public --permanent --add-port=7180/tcp
firewall-cmd --zone=public --permanent --add-port=7181/tcp
firewall-cmd --zone=public --permanent --add-port=7182/tcp
firewall-cmd --reload
#How to install mysql-workbench in mysql repo
wget https://dev.mysql.com/get/mysql80-community-release-el8-1.noarch.rpm
sudo rpm -Uvh mysql80-community-release-el8-1.noarch.rpm
sudo yum update -y
sudo yum install -y mysql-workbench-community
#How to install Mysql-Workbench in RHEL via snapd
https://snapcraft.io/install/mysql-workbench-community/rhel
sudo dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
sudo dnf -y upgrade
sudo subscription-manager repos --enable "rhel-*-optional-rpms" --enable "rhel-*-extras-rpms"
sudo yum install -y snapd
sudo systemctl enable --now snapd.socket
sudo ln -s /var/lib/snapd/snap /snap
sudo snap install mysql-workbench-community
#How to install setup Postgresql
https://www.cyberciti.biz/faq/install-and-setup-postgresql-on-rhel-8/
sudo yum module list | grep postgresql
sudo yum install -y @postgresql:10
sudo postgresql-setup --initdb
sudo passwd postgres
sudo systemctl start postgresql
sudo systemctl enable postgresql
sudo -i -u postgres
psql -U posgres flights
vi /var/lib/pgsql/10/data/pg_hba.conf
host all all 127.0.0.1/32 trust
host all all ::1/128 trust
#How to install pgadmin4-web
https://www.pgadmin.org/download/pgadmin-4-rpm/
sudo rpm -i https://ftp.postgresql.org/pub/pgadmin/pgadmin4/yum/pgadmin4-redhat-repo-1-1.noarch.rpm
sudo yum install -y pgadmin4-web
sudo /usr/pgadmin4/bin/setup-web.sh
sudo rpm -e pgadmin4-redhat-repo
#How to install phpMyAdmin in RHEL
https://www.itzgeek.com/how-tos/linux/centos-how-tos/how-to-install-phpmyadmin-on-rhel-8.html
yum install -y php-json php-mbstring \
yum install -y wget php php-pdo php-pecl-zip php-json php-common php-fpm php-mbstring php-cli php-mysqlnd
wget https://files.phpmyadmin.net/phpMyAdmin/4.9.1/phpMyAdmin-4.9.1-all-languages.tar.gz
tar -zxvf phpMyAdmin-4.9.1-all-languages.tar.gz
mv phpMyAdmin-4.9.1-all-languages /usr/share/phpMyAdmin
cp -pr /usr/share/phpMyAdmin/config.sample.inc.php /usr/share/phpMyAdmin/config.inc.php
vi /usr/share/phpMyAdmin/config.inc.php
mysql -u root -p mysql < /usr/share/phpMyAdmin/sql/create_tables.sql
vi /etc/httpd/conf.d/phpMyAdmin.conf
Alias /phpMyAdmin /usr/share/phpMyAdmin
Alias /phpmyadmin /usr/share/phpMyAdmin
<Directory /usr/share/phpMyAdmin/>
AddDefaultCharset UTF-8
<IfModule mod_authz_core.c>
# Apache 2.4
<RequireAny>
Require all granted
</RequireAny>
</IfModule>
<IfModule !mod_authz_core.c>
# Apache 2.2
Order Deny,Allow
Deny from All
Allow from 127.0.0.1
Allow from ::1
</IfModule>
</Directory>
<Directory /usr/share/phpMyAdmin/setup/>
<IfModule mod_authz_core.c>
# Apache 2.4
<RequireAny>
Require all granted
</RequireAny>
</IfModule>
<IfModule !mod_authz_core.c>
# Apache 2.2
Order Deny,Allow
Deny from All
Allow from 127.0.0.1
Allow from ::1
</IfModule>
</Directory>
mkdir /usr/share/phpMyAdmin/tmp
chmod 777 /usr/share/phpMyAdmin/tmp
chown -R apache:apache /usr/share/phpMyAdmin
systemctl restart httpd
http://localhost/phpMyAdmin
############################################################################
How to create database in Postgre, outside the post postgre console
############################################################################
createdb --username=postgres --host=localhost --password sakila
pg_restore --username=postgres --host=localhost --password --dbname=sakila < sakila_pgsql_dump.tar
pg_dump --username=postgres --host=localhost --password --dbname=sakila --table=store --format=plain > sakila_store_pgsql_dump.sql
pg_restore --username=postgres --host=localhost --password --dbname=sakila < sakila_pgsql_dump.tar
postgres=# \connect sakila;
\include sakila_pgsql_dump.sql;
############################################################################
How to install RHEL 8 in WSL2 with GNOME via VNC Client
############################################################################
Google "RHWSL8 github"
wsl -s RHWSL8 --version 2
wsl -d RHWSL8 genie -s
#Login your RHEL credential#
--
yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
yum remove -y firewalld
sudo systemctl mask --now firewalld
yum remove -y iptables
sudo systemctl mask --now iptables
yum group install GNOME base-x -y
systemctl unmask systemd-logind
#Download the old TigerVNC from the link,
#Installation of old version of TigerVNC
wget https://homes.esat.kuleuven.be/~rtheys/vnc/tigervnc-1.11.0-5.el8.esat~esat1.src.rpm \
wget https://homes.esat.kuleuven.be/~rtheys/vnc/tigervnc-1.11.0-5.el8.esat~esat1.x86_64.rpm \
wget https://homes.esat.kuleuven.be/~rtheys/vnc/tigervnc-icons-1.11.0-5.el8.esat~esat1.noarch.rpm \
wget https://homes.esat.kuleuven.be/~rtheys/vnc/tigervnc-license-1.11.0-5.el8.esat~esat1.noarch.rpm \
wget https://homes.esat.kuleuven.be/~rtheys/vnc/tigervnc-selinux-1.11.0-5.el8.esat~esat1.noarch.rpm \
wget https://homes.esat.kuleuven.be/~rtheys/vnc/tigervnc-server-1.11.0-5.el8.esat~esat1.x86_64.rpm \
wget https://homes.esat.kuleuven.be/~rtheys/vnc/tigervnc-server-minimal-1.11.0-5.el8.esat~esat1.x86_64.rpm \
wget https://homes.esat.kuleuven.be/~rtheys/vnc/tigervnc-server-module-1.11.0-5.el8.esat~esat1.x86_64.rpm
#Step by step to install VNC working in working in RHWSL8
yum install -y \
tigervnc-1.11.0-5.el8.esat~esat1.src.rpm \
tigervnc-1.11.0-5.el8.esat~esat1.x86_64.rpm \
tigervnc-icons-1.11.0-5.el8.esat~esat1.noarch.rpm \
tigervnc-license-1.11.0-5.el8.esat~esat1.noarch.rpm \
tigervnc-selinux-1.11.0-5.el8.esat~esat1.noarch.rpm \
tigervnc-server-1.11.0-5.el8.esat~esat1.x86_64.rpm \
tigervnc-server-minimal-1.11.0-5.el8.esat~esat1.x86_64.rpm \
tigervnc-server-module-1.11.0-5.el8.esat~esat1.x86_64.rpm \
--skip-broken
#Creat vncuser
su - noknok
vncpasswd
#For latest VNC server
cp /usr/lib/systemd/system/vncserver\@.service /etc/systemd/system/vncserver\@.service
vi /etc/systemd/system/vncserver@.service
#Copy this configuration
[Service]
Type=forking
ExecStart=/usr/sbin/runuser -l noknok -c "/usr/bin/vncserver %i -geometry 1366x768"
PIDFile=/home/noknok/.vnc/%H%i.pid
#ExecStart=/usr/libexec/vncsession-start %i
#PIDFile=/run/vncsession-%i.pid
SELinuxContext=system_u:system_r:vnc_session_t:s0
#Restart daemon, so the configuration will take effect
systemctl unmask systemd-logind
systemctl daemon-reload
systemctl start vncserver@:1
systemctl enable vncserver@:1
systemctl status vncserver@:1
#Installation of NXServer
Download NXServer in the website.
nomachine_7.4.1_1_x86_64.rpm
#Before installing nomachine
systemctl unmask systemd-logind
rpm -i nomachine_7.4.1_1_x86_64.rpm
#Check the listening port
netstat -tulpn |grep LISTEN
#Here's the weird part, restart the VNC Service.
systemctl restart vncserver@:1
#It will occur errro, but you can login to GNOMO gui via VNC Client localhost:5901
#or your live ip address with 172.x.x.x
viola, full RED HAT Enterpise Linux without Virtual Machine App.
######################################
#IP Tables
sudo iptables -L
sudo iptables -A INPUT -p tcp --dport 80 -j ACCEPT
service iptables save
systemctl stop firewalld
systemctl disable firewalld
systemctl start iptables
systemctl status iptables
systemctl enable iptables
sudo iptables-save > /etc/sysconfig/iptables
sudo service iptables save
#File transfer in LINUX using scp
scp user@ip:<source_folder>/<file/s> <destination>
######################################
Coursera:
######################################
#how to extract database in Hive
https://medium.com/analytics-vidhya/best-way-to-export-hive-table-to-csv-file-326063f0f229
https://stackoverflow.com/questions/17086642/how-to-export-a-hive-table-into-a-csv-file
#Dumping Database in mysql
mysqldump -u root -p <database> > filename.sql
#Restoring Dataase in mysql
mysqldump -u root -p <created database> < filename.sql
Importing csv into mysql
https://www.coursera.org/learn/introduction-to-relational-databases/lecture/seo5u/loading-data-in-mysql
#Sample Database
https://github.com/words-sdsc/coursera
https://community.ibm.com/community/user/businessanalytics/blogs/steven-macko/2019/07/12/beanie-coffee-1113
https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBM-DB0110EN-SkillsNetwork/datasets/CoffeeData/GeneratedScript.sql
https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBM-DB0110EN-SkillsNetwork/datasets/CoffeeData/CoffeeData.sql
wget https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBM-DB0110EN-SkillsNetwork/datasets/sakila/sakila_mysql_dump.sql \
wget https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBM-DB0110EN-SkillsNetwork/datasets/sakila/sakila_pgsql_dump.sql \
wget https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBM-DB0110EN-SkillsNetwork/datasets/eBooks/eBooks_mysql_dump.sql
wget https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBM-DB0110EN-SkillsNetwork/datasets/sakila/sakila_pgsql_dump.tar \
wget https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBM-DB0110EN-SkillsNetwork/datasets/eBooks/eBooks_pgsql_dump.tar
hive -e 'set hive.cli.print.header=true; select * from fly.planes' | sed 's/[\t]/,/g' > /home/fly.planes.csv
hive -e 'set hive.cli.print.header=true; select * from fly.airlines' | sed 's/[\t]/,/g' > /home/fly.airlines.csv
hive -e 'set hive.cli.print.header=true; select * from your_Table' | sed 's/[\t]/,/g' > /home/yourfile.csv
hive -e 'set hive.cli.print.header=true; select * from your_Table' | sed 's/[\t]/,/g' > /home/yourfile.csv
hadoop fs -cat hdfs://servername/user/hive/warehouse/databasename/table_csv_export_data/* > ~/output.csvCopy
#how to upgrade from CentOS6 into CentOS7
https://itbeginner.net/upgrade-centos-6-7.html
[upg]
name=CentOS-$releasever - Upgrade Tool
baseurl=https://buildlogs.centos.org/centos/6/upg/x86_64/
gpgcheck=1
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6