diff --git a/.vscode/settings.json b/.vscode/settings.json index 0002907..caa4043 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,7 +3,7 @@ "editor.cursorBlinking": "solid", "editor.fontFamily": "ui-monospace, Menlo, Monaco, 'Cascadia Mono', 'Segoe UI Mono', 'Roboto Mono', 'Oxygen Mono', 'Ubuntu Monospace', 'Source Code Pro', 'Fira Mono', 'Droid Sans Mono', 'Courier New', monospace", "editor.fontLigatures": false, - "editor.fontSize": 22, + "editor.fontSize": 15, "editor.formatOnPaste": true, "editor.formatOnSave": true, "editor.lineNumbers": "on", @@ -17,9 +17,11 @@ "explorer.openEditors.visible": 0, "files.autoSave": "afterDelay", "screencastMode.onlyKeyboardShortcuts": true, - "terminal.integrated.fontSize": 18, - "workbench.activityBar.visible": true, + "terminal.integrated.fontSize": 15, "workbench.colorTheme": "Visual Studio Dark", "workbench.fontAliasing": "antialiased", - "workbench.statusBar.visible": true -} + "workbench.statusBar.visible": true, + "githubPullRequests.ignoredPullRequestBranches": [ + "main" + ] +} \ No newline at end of file diff --git a/ch1/01_bash.sh b/ch1/01_bash.sh new file mode 100755 index 0000000..9ed4d9d --- /dev/null +++ b/ch1/01_bash.sh @@ -0,0 +1,21 @@ +# what is Bash? +# Bash is a widly used shell, available on many platforms +# Short for Bourne Again Shell, in reference to the earlier Bourne Shell +# An interactive command-line shell that also allows commands to be combined into script files, which can be run like programs +# Combine commands into scripts save time and reduces errors + +# what is Bash Best For? +# Bash is best for writting small to medium size scripts that run on Linux systems +# If your workflow can be run as commands in Bash terminal, consider making a script +# If your workflow requires other more specialized tools, Bash may not be the best choice for automation + +bash --version + +# GNU bash, version 5.0.17(1)-release (x86_64-pc-linux-gnu) +# Copyright (C) 2019 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later + +# This is free software; you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. + +echo $SHELL diff --git a/ch1/02_Pipes_and_redirections.sh b/ch1/02_Pipes_and_redirections.sh new file mode 100755 index 0000000..a95f436 --- /dev/null +++ b/ch1/02_Pipes_and_redirections.sh @@ -0,0 +1,42 @@ +#1/bin/bash + +# Pipes send the output of one process to another + +# cat command +# cat /workspaces/learning-bash-scripting-3212393/dir1/lorem.txt + +cat /workspaces/learning-bash-scripting-3212393/dir1/lorem.txt | wc + +# output +# 45 1853 12577 +# 45 - line +# 1853 - words +# 12577 - charcters + +# we can string many command with | (like grep, awk, sed, cut) + +# ls command +# pipes send the output of one process to another +ls | wc -l + +# Redirections send streams (standard input, output, and error) to or from files + +ls > list.txt + +# Redirection + +# stream Name Conctent +# 0 Standard input(stdin) Keyboard or other input +# 1 Standard output(stdout) Regular output +# 2 Standard error(stderr) Output marked as 'error' + +# symbol Function +# > Output redirection(truncate) +# >> Output redirection(append) +# < Input redirection +# << Here document + +ls /loram +ls /loram 1>output.txt 2>error.txt + +cat << EndOfText diff --git a/ch1/03_Bash_built-ins_and_other_commands.sh b/ch1/03_Bash_built-ins_and_other_commands.sh new file mode 100755 index 0000000..b6db391 --- /dev/null +++ b/ch1/03_Bash_built-ins_and_other_commands.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Commands and Built-ins + +# Many of the programes we use in Bash are commands +# Seperate software that does not depend on Bash + +# Bash includes build-in commands as well +# Part of Bash itself + +# Some Bash built-ins have the same name as other commands +# See the documention for a full list of built-ins + +echo hello there +printf hello + +command echo hello +builtin echo hello + +command -V df +# df is /usr/bin/df + +command -V echo +enable -n echo + +command -V echo +# echo is a shell builtin + +enable -n echo +enable echo + +# Supporting information for built-in +help echo \ No newline at end of file diff --git a/ch1/04_Brackets_and_braces_in_Bash.sh b/ch1/04_Brackets_and_braces_in_Bash.sh new file mode 100644 index 0000000..c23ca1c --- /dev/null +++ b/ch1/04_Brackets_and_braces_in_Bash.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# () - Parentheses +# {} - Braces +# [] - Bracets + +# Bash uses these characters differently than other languages. +# In Bash Parentheses, Braces & Bracets act as commands diff --git a/ch1/05_Bash_expansions_and_substitutions.sh b/ch1/05_Bash_expansions_and_substitutions.sh new file mode 100644 index 0000000..10f70c1 --- /dev/null +++ b/ch1/05_Bash_expansions_and_substitutions.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Expansions and subtitutions allow us to specify values that aren't know until a script runs. + + +# Representation Name +# ~ Tilde expansion +# {...} Brace expansion +# ${...} Parameter expansion +# $(...) Command substitution +# $((...)) Arithmetic expansion + +# ~ tilde expansion represents the user's #HOME environment variable. + +echo ~ +# /home/codespace + +whoami +# codespace + +echo ~- +# /workspaces/learning-bash-scripting-3212393 \ No newline at end of file diff --git a/ch1/06_Bash_expansion.sh b/ch1/06_Bash_expansion.sh new file mode 100644 index 0000000..626572c --- /dev/null +++ b/ch1/06_Bash_expansion.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Brace Expansion + +echo /tmp/{one,two,three}/file.txt +# /tmp/one/file.txt /tmp/two/file.txt /tmp/three/file.txt + +echo c{a,o,u}t +# cat cot cut + +echo /tmp/{1..3}/file.txt +# /tmp/1/file.txt /tmp/2/file.txt /tmp/3/file.txt + +echo {00..100} +# 000 001 002 003 004 005 006 007 008 009 010 011 012 013 014 015 016 017 018 019 020 021 022 023 024 025 026 027 028 029 030 031 032 033 034 035 036 037 038 039 040 041 042 043 044 045 046 047 048 049 050 051 052 053 054 055 056 057 058 059 060 061 062 063 064 065 066 067 068 069 070 071 072 073 074 075 076 077 078 079 080 081 082 083 084 085 086 087 088 089 090 091 092 093 094 095 096 097 098 099 100 + +echo {1..10} +# 1 2 3 4 5 6 7 8 9 10 + +echo {a..z} +# a b c d e f g h i j k l m n o p q r s t u v w x y z + +echo {A..Z} +# A B C D E F G H I J K L M N O P Q R S T U V W X Y Z + +echo {1..30..3} +# 1 4 7 10 13 16 19 22 25 28 + +echo {a..z..2} +# a c e g i k m o q s u w y + +touch file_{01..12}{a..d} +ls + +# 01_bash.sh file_03c file_08b +# 02_Pipes_and_redirections.sh file_03d file_08c +# 03_Bash_built-ins_and_other_commands.sh file_04a file_08d +# 04_Brackets_and_braces_in_Bash.sh file_04b file_09a +# 05_Bash_expansions_and_substitutions.sh file_04c file_09b +# 06_Bash_expansion.sh file_04d file_09c +# 07_Parameter_expansion.sh file_05a file_09d +# 08_Command_substitution.sh file_05b file_10a +# 09_Arithmetic_expansion.sh file_05c file_10b +# file_01a file_05d file_10c +# file_01b file_06a file_10d +# file_01c file_06b file_11a +# file_01d file_06c file_11b +# file_02a file_06d file_11c +# file_02b file_07a file_11d +# file_02c file_07b file_12a +# file_02d file_07c file_12b +# file_03a file_07d file_12c +# file_03b file_08a file_12d + +rm file_* + +echo {cat,dog,fox}_{1..5} +# cat_1 cat_2 cat_3 cat_4 cat_5 dog_1 dog_2 dog_3 dog_4 dog_5 fox_1 fox_2 fox_3 fox_4 fox_5 + +# To view first line of file each directory +head -n1 ../{dir1,dir2,dir3}/lorem.txt + diff --git a/ch1/07_Parameter_expansion.sh b/ch1/07_Parameter_expansion.sh new file mode 100644 index 0000000..7f03324 --- /dev/null +++ b/ch1/07_Parameter_expansion.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# ${...} +# Parameter expansion retrives and transforms stored values. + +a="Hello World" + +echo $a +# Hello World + +echo ${a} +# Hello World + +echo ${a:6} +# World + +echo ${a:1:9} +# ello Worl + +# Replace the words +echo ${a/World/Everbody} +# Hello Everbody + +greeting="hello there!" +echo ${greeting//e/_} +# h_llo th_r_! + +echo ${greeting/e/_} +# h_llo there! + +echo $greeting:4:3 +# hello there!:4:3 \ No newline at end of file diff --git a/ch1/08_Command_substitution.sh b/ch1/08_Command_substitution.sh new file mode 100644 index 0000000..a8d8eae --- /dev/null +++ b/ch1/08_Command_substitution.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# $(...) +# Command substitution puts the output of one command inside another. +# Older representation: `...` + +$ uname -r +# 6.2.0-1018-azure + +$ echo "The kernal is $(uname -r)." +# The kernal is 6.2.0-1018-azure. + +$ echo "The Python version is $(python -V)." +# The Python version is Python 3.10.8. + +# Transform the lower to upper case! +$ echo "Result: $(python3 -c 'print("Hello from python!")' | tr [a-z] [A-Z])'" +# Result: HELLO FROM PYTHON!' \ No newline at end of file diff --git a/ch1/09_Arithmetic_expansion.sh b/ch1/09_Arithmetic_expansion.sh new file mode 100644 index 0000000..9a2f65e --- /dev/null +++ b/ch1/09_Arithmetic_expansion.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# $((...)) +# Arithmetic expansion does calculations. +# Older representation:$[...] + +echo $((2+2)) +# 4 + +echo $((3-2)) +# 1 + +echo $((4*5)) +# 25 + +# Bash can +echo $((4/5)) +# 0 +# Bash can only calculate integer + +# For finding reminder part use mod operator % +echo $((5%3)) +# 2 + + +# Older representation +echo $[5%3] diff --git a/use_case/ava_data.sh b/use_case/ava_data.sh new file mode 100755 index 0000000..b996bc7 --- /dev/null +++ b/use_case/ava_data.sh @@ -0,0 +1,36 @@ +#!/bin/bash +app_home="/workspaces/learning-bash-scripting-3212393/use_case" +sql_file="sample.sql" +frequency=${1} +start_date=${2} +end_date=${3} + + + +if [[ -z ${frequency} ]] +then + echo "please provide the frequency" + exit 1 +fi + +if [[ ${frequency} == "custom" ]] && ([[ -z ${start_date} ]] || [[ -z ${end_date} ]]) +then + echo "Please provide the start_date and end_date" + exit 1 +fi + +if [[ ${frequency} == 'monthly' ]] +then + start_date=$(date -d "$(date -d "$($date) -1 month" +%Y-%m-01)" +%Y-%m-%d) + end_date=$(date -d "$(date -d "$start_date +1 month" +%Y-%m-01) -1sec" +%F) +fi + +if [[ ${FREQUENCY} == 'custom' ]] +then + start_date=$(date -d $start_date +%Y-%m-%d) + end_date=$(date -d $end_date +%Y-%m-%d) +fi + +var="'$start_date' and '$end_date'" +echo "Avalara ${FREQUENCY} recon for date_key between $var" +sed -i -e "s/\(date_key between \).*/\1$var/" ${app_home}/${sql_file} \ No newline at end of file diff --git a/use_case/date_test.sh b/use_case/date_test.sh new file mode 100755 index 0000000..9623338 --- /dev/null +++ b/use_case/date_test.sh @@ -0,0 +1,36 @@ +#!/bin/bash +APP_HOME="/workspaces/learning-bash-scripting-3212393/use-case" +SQL_FILE="sample.sql" +START_DATE="2023-01-01" +END_DATE="2023-12-18" + +START_DATE=$(date -d $START_DATE +%Y-%m-%d) +END_DATE=$(date -d $END_DATE +%Y-%m-%d) +LOCAL_START_DATE=$START_DATE + +START_MONTH=$(date -d $START_DATE +%Y-%m) +END_MONTH=$(date -d $END_DATE +%Y-%m) +LOCAL_START_MONTH=$START_MONTH + +echo $LOCAL_START_MONTH +echo $END_MONTH + +THE_START_DATE=$LOCAL_START_DATE +echo $THE_START_DATE + +THE_END_DATE=$(date -d "$(date -d "$THE_START_DATE +1 month" +%Y-%m-01) -1sec" +%F) +echo $THE_END_DATE + +THE_START_DATE=$(date -d "$THE_END_DATE +1 days" +%Y-%m-01) +echo $THE_START_DATE + +THE_END_DATE=$(date -d "$(date -d "$THE_START_DATE +1 month" +%Y-%m-01) -1sec" +%F) +echo $THE_END_DATE + +LOCAL_START_MONTH=$(date -d $THE_START_DATE +%Y-%m) +echo $LOCAL_START_MONTH +echo $END_MONTH + + + + diff --git a/use_case/for-loops.sh b/use_case/for-loops.sh new file mode 100755 index 0000000..17a61a8 --- /dev/null +++ b/use_case/for-loops.sh @@ -0,0 +1,24 @@ +#!/bin/bash +start_date='2023-01-01' +end_date='2023-12-31' + +date_diff=$((($(date -d $end_date "+%s") - $(date -d $start_date "+%s")) / 86400)) +months=$(($date_diff/30)) +days=$((5%3)) +echo $date_diff +if [[ $date_diff > 30 ]] +then + echo $end_date +else + echo "date is less" +fi + +#echo $months +#echo $days + +START_LAST_MONTH=$(date "+%F" -d "$(date +'%Y-%m-01') 0 month") +END_LAST_MONTH=$(date "+%F" -d "$START_LAST_MONTH +1 month 0 day") + +#Test Code +echo START_LAST_MONTH=$START_LAST_MONTH +echo END_LAST_MONTH=$END_LAST_MONTH \ No newline at end of file diff --git a/use_case/rm_file.sh b/use_case/rm_file.sh new file mode 100755 index 0000000..e329be6 --- /dev/null +++ b/use_case/rm_file.sh @@ -0,0 +1,21 @@ +#!/bin/bash +APP_HOME=/workspaces/learning-bash-scripting-3212393 +SH_LOG_DIR=${APP_HOME}/use_case +LOG_FILE1="transaction_count_bdp_backfill_logging_output" + +RM_FILE="${SH_LOG_DIR}/${LOG_FILE1}-$(date +%Y-%m-%d).txt" + +RM_FILE_LINE_COUNT=$(head -n 3 -- "$RM_FILE" | wc -w) + +echo $RM_FILE_LINE_COUNT +if [[ ${RM_FILE_LINE_COUNT} -eq 0 ]] +then + rm -rf ${RM_FILE} + echo "${RM_FILE} empty log file removed" +elif [[ ${RM_FILE_LINE_COUNT} -gt 0 ]] +then + echo "${RM_FILE} log file is not empty" +elif [[ -f ${RM_FILE} ]] +then + echo "${RM_FILE} log file not found" +fi \ No newline at end of file diff --git a/use_case/sample.sql b/use_case/sample.sql new file mode 100644 index 0000000..9094f30 --- /dev/null +++ b/use_case/sample.sql @@ -0,0 +1,5 @@ +select * +from employees +where date_key between '2024-01-01' and '2024-01-31' +where transaction_date_key between '2024-01-01' and '2024-01-31' +where cast(date_add(cast(n.reporting_date_key as date), -1) as varchar(15)) between '2024-01-01' and '2024-01-31' \ No newline at end of file diff --git a/use_case/sed-test.sh b/use_case/sed-test.sh new file mode 100755 index 0000000..8e60ed4 --- /dev/null +++ b/use_case/sed-test.sh @@ -0,0 +1,33 @@ +#!/bin/bash +APP_HOME="/workspaces/Linux-learning-bash-scripting/use_case" +SQL_FILE="sample.sql" +START_DATE=${1} +END_DATE=${2} + +START_DATE=$(date -d $START_DATE +%Y-%m-%d) +END_DATE=$(date -d $END_DATE +%Y-%m-%d) +LOCAL_START_DATE=$START_DATE + +START_MONTH=$(date -d $START_DATE +%Y-%m) +END_MONTH=$(date -d $END_DATE +%Y-%m) +LOCAL_START_MONTH=$START_MONTH + +if [ $START_MONTH == $END_MONTH ] +then + VAR="'$START_DATE' and '$END_DATE'" + sed -i -e "s/\(between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} +else + while [ $LOCAL_START_MONTH != $END_MONTH ] + do + THE_START_DATE=$LOCAL_START_DATE + THE_END_DATE=$(date -d "$(date -d "$THE_START_DATE +1 month" +%Y-%m-01) -1sec" +%F) + + VAR="'$THE_START_DATE' and '$THE_END_DATE'" + sed -i -e "s/\(between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} + LOCAL_START_DATE=$(date -d "$THE_END_DATE +1 days" +%Y-%m-01) + LOCAL_START_MONTH=$(date -d $LOCAL_START_DATE +%Y-%m) + done + + VAR="'$LOCAL_START_DATE' and '$END_DATE'" + sed -i -e "s/\(between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} +fi \ No newline at end of file diff --git a/use_case/transaction_count_bdp_backfill_logging_output-2023-12-21.txt b/use_case/transaction_count_bdp_backfill_logging_output-2023-12-21.txt new file mode 100644 index 0000000..76afcce --- /dev/null +++ b/use_case/transaction_count_bdp_backfill_logging_output-2023-12-21.txt @@ -0,0 +1,2 @@ +test +test \ No newline at end of file diff --git a/use_case/unix.txt b/use_case/unix.txt new file mode 100644 index 0000000..76afcce --- /dev/null +++ b/use_case/unix.txt @@ -0,0 +1,2 @@ +test +test \ No newline at end of file