From cc983858fefb0832132f1ff23dad53f6f6a3a07c Mon Sep 17 00:00:00 2001 From: Sivakumar R <108018740+sivakumar-dr@users.noreply.github.com> Date: Mon, 27 Nov 2023 06:06:56 +0000 Subject: [PATCH 01/11] modified font size --- .vscode/settings.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 0002907..cf4d9bd 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,7 +3,7 @@ "editor.cursorBlinking": "solid", "editor.fontFamily": "ui-monospace, Menlo, Monaco, 'Cascadia Mono', 'Segoe UI Mono', 'Roboto Mono', 'Oxygen Mono', 'Ubuntu Monospace', 'Source Code Pro', 'Fira Mono', 'Droid Sans Mono', 'Courier New', monospace", "editor.fontLigatures": false, - "editor.fontSize": 22, + "editor.fontSize": 16, "editor.formatOnPaste": true, "editor.formatOnSave": true, "editor.lineNumbers": "on", @@ -22,4 +22,4 @@ "workbench.colorTheme": "Visual Studio Dark", "workbench.fontAliasing": "antialiased", "workbench.statusBar.visible": true -} +} \ No newline at end of file From 61878e0ad89a68cc80859a64e03750d98e9aff4a Mon Sep 17 00:00:00 2001 From: Sivakumar R <108018740+sivakumar-dr@users.noreply.github.com> Date: Tue, 19 Dec 2023 13:01:47 +0000 Subject: [PATCH 02/11] json file modified --- .vscode/settings.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index cf4d9bd..7bbf35a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,8 +17,7 @@ "explorer.openEditors.visible": 0, "files.autoSave": "afterDelay", "screencastMode.onlyKeyboardShortcuts": true, - "terminal.integrated.fontSize": 18, - "workbench.activityBar.visible": true, + "terminal.integrated.fontSize": 16, "workbench.colorTheme": "Visual Studio Dark", "workbench.fontAliasing": "antialiased", "workbench.statusBar.visible": true From 6c397ef1508cd9b3c647cf344e3bdda30ab68755 Mon Sep 17 00:00:00 2001 From: Sivakumar R <108018740+sivakumar-dr@users.noreply.github.com> Date: Tue, 19 Dec 2023 13:02:16 +0000 Subject: [PATCH 03/11] json file modified --- .vscode/settings.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 7bbf35a..31f3a83 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -20,5 +20,8 @@ "terminal.integrated.fontSize": 16, "workbench.colorTheme": "Visual Studio Dark", "workbench.fontAliasing": "antialiased", - "workbench.statusBar.visible": true + "workbench.statusBar.visible": true, + "githubPullRequests.ignoredPullRequestBranches": [ + "main" + ] } \ No newline at end of file From 40458c137ad75ec0dde7a2116971175311aff940 Mon Sep 17 00:00:00 2001 From: Sivakumar R <108018740+sivakumar-dr@users.noreply.github.com> Date: Tue, 19 Dec 2023 13:11:26 +0000 Subject: [PATCH 04/11] for loops testing --- loops/for-loops.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 loops/for-loops.sh diff --git a/loops/for-loops.sh b/loops/for-loops.sh new file mode 100644 index 0000000..7e979d2 --- /dev/null +++ b/loops/for-loops.sh @@ -0,0 +1,2 @@ +#!/bin/bash +echo 'hello world' From 8cb95238485a8b8cca3d32fa3f6ed5065acfba1b Mon Sep 17 00:00:00 2001 From: Sivakumar R <108018740+sivakumar-dr@users.noreply.github.com> Date: Tue, 19 Dec 2023 18:08:12 +0000 Subject: [PATCH 05/11] chapter 1 exercises --- .vscode/settings.json | 4 +- ch1/01_bash.sh | 21 +++++++ ch1/02_Pipes_and_redirections.sh | 42 ++++++++++++++ ch1/03_Bash_built-ins_and_other_commands.sh | 32 +++++++++++ ch1/04_Brackets_and_braces_in_Bash.sh | 8 +++ ch1/05_Bash_expansions_and_substitutions.sh | 22 ++++++++ ch1/06_Bash_expansion.sh | 61 +++++++++++++++++++++ ch1/07_Parameter_expansion.sh | 32 +++++++++++ ch1/08_Command_substitution.sh | 18 ++++++ ch1/09_Arithmetic_expansion.sh | 27 +++++++++ loops/for-loops.sh | 24 +++++++- 11 files changed, 288 insertions(+), 3 deletions(-) create mode 100755 ch1/01_bash.sh create mode 100755 ch1/02_Pipes_and_redirections.sh create mode 100755 ch1/03_Bash_built-ins_and_other_commands.sh create mode 100644 ch1/04_Brackets_and_braces_in_Bash.sh create mode 100644 ch1/05_Bash_expansions_and_substitutions.sh create mode 100644 ch1/06_Bash_expansion.sh create mode 100644 ch1/07_Parameter_expansion.sh create mode 100644 ch1/08_Command_substitution.sh create mode 100644 ch1/09_Arithmetic_expansion.sh mode change 100644 => 100755 loops/for-loops.sh diff --git a/.vscode/settings.json b/.vscode/settings.json index 31f3a83..caa4043 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,7 +3,7 @@ "editor.cursorBlinking": "solid", "editor.fontFamily": "ui-monospace, Menlo, Monaco, 'Cascadia Mono', 'Segoe UI Mono', 'Roboto Mono', 'Oxygen Mono', 'Ubuntu Monospace', 'Source Code Pro', 'Fira Mono', 'Droid Sans Mono', 'Courier New', monospace", "editor.fontLigatures": false, - "editor.fontSize": 16, + "editor.fontSize": 15, "editor.formatOnPaste": true, "editor.formatOnSave": true, "editor.lineNumbers": "on", @@ -17,7 +17,7 @@ "explorer.openEditors.visible": 0, "files.autoSave": "afterDelay", "screencastMode.onlyKeyboardShortcuts": true, - "terminal.integrated.fontSize": 16, + "terminal.integrated.fontSize": 15, "workbench.colorTheme": "Visual Studio Dark", "workbench.fontAliasing": "antialiased", "workbench.statusBar.visible": true, diff --git a/ch1/01_bash.sh b/ch1/01_bash.sh new file mode 100755 index 0000000..9ed4d9d --- /dev/null +++ b/ch1/01_bash.sh @@ -0,0 +1,21 @@ +# what is Bash? +# Bash is a widly used shell, available on many platforms +# Short for Bourne Again Shell, in reference to the earlier Bourne Shell +# An interactive command-line shell that also allows commands to be combined into script files, which can be run like programs +# Combine commands into scripts save time and reduces errors + +# what is Bash Best For? +# Bash is best for writting small to medium size scripts that run on Linux systems +# If your workflow can be run as commands in Bash terminal, consider making a script +# If your workflow requires other more specialized tools, Bash may not be the best choice for automation + +bash --version + +# GNU bash, version 5.0.17(1)-release (x86_64-pc-linux-gnu) +# Copyright (C) 2019 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later + +# This is free software; you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. + +echo $SHELL diff --git a/ch1/02_Pipes_and_redirections.sh b/ch1/02_Pipes_and_redirections.sh new file mode 100755 index 0000000..a95f436 --- /dev/null +++ b/ch1/02_Pipes_and_redirections.sh @@ -0,0 +1,42 @@ +#1/bin/bash + +# Pipes send the output of one process to another + +# cat command +# cat /workspaces/learning-bash-scripting-3212393/dir1/lorem.txt + +cat /workspaces/learning-bash-scripting-3212393/dir1/lorem.txt | wc + +# output +# 45 1853 12577 +# 45 - line +# 1853 - words +# 12577 - charcters + +# we can string many command with | (like grep, awk, sed, cut) + +# ls command +# pipes send the output of one process to another +ls | wc -l + +# Redirections send streams (standard input, output, and error) to or from files + +ls > list.txt + +# Redirection + +# stream Name Conctent +# 0 Standard input(stdin) Keyboard or other input +# 1 Standard output(stdout) Regular output +# 2 Standard error(stderr) Output marked as 'error' + +# symbol Function +# > Output redirection(truncate) +# >> Output redirection(append) +# < Input redirection +# << Here document + +ls /loram +ls /loram 1>output.txt 2>error.txt + +cat << EndOfText diff --git a/ch1/03_Bash_built-ins_and_other_commands.sh b/ch1/03_Bash_built-ins_and_other_commands.sh new file mode 100755 index 0000000..b6db391 --- /dev/null +++ b/ch1/03_Bash_built-ins_and_other_commands.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Commands and Built-ins + +# Many of the programes we use in Bash are commands +# Seperate software that does not depend on Bash + +# Bash includes build-in commands as well +# Part of Bash itself + +# Some Bash built-ins have the same name as other commands +# See the documention for a full list of built-ins + +echo hello there +printf hello + +command echo hello +builtin echo hello + +command -V df +# df is /usr/bin/df + +command -V echo +enable -n echo + +command -V echo +# echo is a shell builtin + +enable -n echo +enable echo + +# Supporting information for built-in +help echo \ No newline at end of file diff --git a/ch1/04_Brackets_and_braces_in_Bash.sh b/ch1/04_Brackets_and_braces_in_Bash.sh new file mode 100644 index 0000000..c23ca1c --- /dev/null +++ b/ch1/04_Brackets_and_braces_in_Bash.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# () - Parentheses +# {} - Braces +# [] - Bracets + +# Bash uses these characters differently than other languages. +# In Bash Parentheses, Braces & Bracets act as commands diff --git a/ch1/05_Bash_expansions_and_substitutions.sh b/ch1/05_Bash_expansions_and_substitutions.sh new file mode 100644 index 0000000..10f70c1 --- /dev/null +++ b/ch1/05_Bash_expansions_and_substitutions.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Expansions and subtitutions allow us to specify values that aren't know until a script runs. + + +# Representation Name +# ~ Tilde expansion +# {...} Brace expansion +# ${...} Parameter expansion +# $(...) Command substitution +# $((...)) Arithmetic expansion + +# ~ tilde expansion represents the user's #HOME environment variable. + +echo ~ +# /home/codespace + +whoami +# codespace + +echo ~- +# /workspaces/learning-bash-scripting-3212393 \ No newline at end of file diff --git a/ch1/06_Bash_expansion.sh b/ch1/06_Bash_expansion.sh new file mode 100644 index 0000000..626572c --- /dev/null +++ b/ch1/06_Bash_expansion.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Brace Expansion + +echo /tmp/{one,two,three}/file.txt +# /tmp/one/file.txt /tmp/two/file.txt /tmp/three/file.txt + +echo c{a,o,u}t +# cat cot cut + +echo /tmp/{1..3}/file.txt +# /tmp/1/file.txt /tmp/2/file.txt /tmp/3/file.txt + +echo {00..100} +# 000 001 002 003 004 005 006 007 008 009 010 011 012 013 014 015 016 017 018 019 020 021 022 023 024 025 026 027 028 029 030 031 032 033 034 035 036 037 038 039 040 041 042 043 044 045 046 047 048 049 050 051 052 053 054 055 056 057 058 059 060 061 062 063 064 065 066 067 068 069 070 071 072 073 074 075 076 077 078 079 080 081 082 083 084 085 086 087 088 089 090 091 092 093 094 095 096 097 098 099 100 + +echo {1..10} +# 1 2 3 4 5 6 7 8 9 10 + +echo {a..z} +# a b c d e f g h i j k l m n o p q r s t u v w x y z + +echo {A..Z} +# A B C D E F G H I J K L M N O P Q R S T U V W X Y Z + +echo {1..30..3} +# 1 4 7 10 13 16 19 22 25 28 + +echo {a..z..2} +# a c e g i k m o q s u w y + +touch file_{01..12}{a..d} +ls + +# 01_bash.sh file_03c file_08b +# 02_Pipes_and_redirections.sh file_03d file_08c +# 03_Bash_built-ins_and_other_commands.sh file_04a file_08d +# 04_Brackets_and_braces_in_Bash.sh file_04b file_09a +# 05_Bash_expansions_and_substitutions.sh file_04c file_09b +# 06_Bash_expansion.sh file_04d file_09c +# 07_Parameter_expansion.sh file_05a file_09d +# 08_Command_substitution.sh file_05b file_10a +# 09_Arithmetic_expansion.sh file_05c file_10b +# file_01a file_05d file_10c +# file_01b file_06a file_10d +# file_01c file_06b file_11a +# file_01d file_06c file_11b +# file_02a file_06d file_11c +# file_02b file_07a file_11d +# file_02c file_07b file_12a +# file_02d file_07c file_12b +# file_03a file_07d file_12c +# file_03b file_08a file_12d + +rm file_* + +echo {cat,dog,fox}_{1..5} +# cat_1 cat_2 cat_3 cat_4 cat_5 dog_1 dog_2 dog_3 dog_4 dog_5 fox_1 fox_2 fox_3 fox_4 fox_5 + +# To view first line of file each directory +head -n1 ../{dir1,dir2,dir3}/lorem.txt + diff --git a/ch1/07_Parameter_expansion.sh b/ch1/07_Parameter_expansion.sh new file mode 100644 index 0000000..7f03324 --- /dev/null +++ b/ch1/07_Parameter_expansion.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# ${...} +# Parameter expansion retrives and transforms stored values. + +a="Hello World" + +echo $a +# Hello World + +echo ${a} +# Hello World + +echo ${a:6} +# World + +echo ${a:1:9} +# ello Worl + +# Replace the words +echo ${a/World/Everbody} +# Hello Everbody + +greeting="hello there!" +echo ${greeting//e/_} +# h_llo th_r_! + +echo ${greeting/e/_} +# h_llo there! + +echo $greeting:4:3 +# hello there!:4:3 \ No newline at end of file diff --git a/ch1/08_Command_substitution.sh b/ch1/08_Command_substitution.sh new file mode 100644 index 0000000..a8d8eae --- /dev/null +++ b/ch1/08_Command_substitution.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# $(...) +# Command substitution puts the output of one command inside another. +# Older representation: `...` + +$ uname -r +# 6.2.0-1018-azure + +$ echo "The kernal is $(uname -r)." +# The kernal is 6.2.0-1018-azure. + +$ echo "The Python version is $(python -V)." +# The Python version is Python 3.10.8. + +# Transform the lower to upper case! +$ echo "Result: $(python3 -c 'print("Hello from python!")' | tr [a-z] [A-Z])'" +# Result: HELLO FROM PYTHON!' \ No newline at end of file diff --git a/ch1/09_Arithmetic_expansion.sh b/ch1/09_Arithmetic_expansion.sh new file mode 100644 index 0000000..9a2f65e --- /dev/null +++ b/ch1/09_Arithmetic_expansion.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# $((...)) +# Arithmetic expansion does calculations. +# Older representation:$[...] + +echo $((2+2)) +# 4 + +echo $((3-2)) +# 1 + +echo $((4*5)) +# 25 + +# Bash can +echo $((4/5)) +# 0 +# Bash can only calculate integer + +# For finding reminder part use mod operator % +echo $((5%3)) +# 2 + + +# Older representation +echo $[5%3] diff --git a/loops/for-loops.sh b/loops/for-loops.sh old mode 100644 new mode 100755 index 7e979d2..17a61a8 --- a/loops/for-loops.sh +++ b/loops/for-loops.sh @@ -1,2 +1,24 @@ #!/bin/bash -echo 'hello world' +start_date='2023-01-01' +end_date='2023-12-31' + +date_diff=$((($(date -d $end_date "+%s") - $(date -d $start_date "+%s")) / 86400)) +months=$(($date_diff/30)) +days=$((5%3)) +echo $date_diff +if [[ $date_diff > 30 ]] +then + echo $end_date +else + echo "date is less" +fi + +#echo $months +#echo $days + +START_LAST_MONTH=$(date "+%F" -d "$(date +'%Y-%m-01') 0 month") +END_LAST_MONTH=$(date "+%F" -d "$START_LAST_MONTH +1 month 0 day") + +#Test Code +echo START_LAST_MONTH=$START_LAST_MONTH +echo END_LAST_MONTH=$END_LAST_MONTH \ No newline at end of file From f89a4d96783458060b4cd3bd841d244faf08af23 Mon Sep 17 00:00:00 2001 From: Sivakumar R <108018740+sivakumar-dr@users.noreply.github.com> Date: Wed, 20 Dec 2023 13:36:32 +0000 Subject: [PATCH 06/11] Usecase testing done --- {loops => use-case}/for-loops.sh | 0 use-case/sample.sql | 3 +++ use-case/sed-test.sh | 19 +++++++++++++++++++ 3 files changed, 22 insertions(+) rename {loops => use-case}/for-loops.sh (100%) create mode 100644 use-case/sample.sql create mode 100755 use-case/sed-test.sh diff --git a/loops/for-loops.sh b/use-case/for-loops.sh similarity index 100% rename from loops/for-loops.sh rename to use-case/for-loops.sh diff --git a/use-case/sample.sql b/use-case/sample.sql new file mode 100644 index 0000000..0a06e04 --- /dev/null +++ b/use-case/sample.sql @@ -0,0 +1,3 @@ +select * +from employees +where date_key between '2023-12-01' and '2023-12-20' \ No newline at end of file diff --git a/use-case/sed-test.sh b/use-case/sed-test.sh new file mode 100755 index 0000000..08d19b1 --- /dev/null +++ b/use-case/sed-test.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +START_DATE="2023-12-01" +END_DATE="2023-12-18" + +START_MONTH=$(date -d "$START_DATE" +%Y-%m) +END_MONTH=$(date -d "$END_DATE" +%Y-%m) + +if [ $START_MONTH == $END_MONTH ] +then + START_DATE=$START_DATE + END_DATE=$END_DATE +else + START_DATE="null" + END_DATE="null" +fi + +echo $START_DATE +echo $END_DATE \ No newline at end of file From 5b024ddbda0b7cd30615af8e9dfb5a6ec1dd3123 Mon Sep 17 00:00:00 2001 From: Sivakumar R <108018740+sivakumar-dr@users.noreply.github.com> Date: Wed, 20 Dec 2023 16:38:24 +0000 Subject: [PATCH 07/11] sed testing --- use-case/sample.sql | 2 +- use-case/sed-test.sh | 33 ++++++++++++++++++++++++++------- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/use-case/sample.sql b/use-case/sample.sql index 0a06e04..e7a08a7 100644 --- a/use-case/sample.sql +++ b/use-case/sample.sql @@ -1,3 +1,3 @@ select * from employees -where date_key between '2023-12-01' and '2023-12-20' \ No newline at end of file +where date_key between '2023-12-01' and '2023-12-18' \ No newline at end of file diff --git a/use-case/sed-test.sh b/use-case/sed-test.sh index 08d19b1..49906aa 100755 --- a/use-case/sed-test.sh +++ b/use-case/sed-test.sh @@ -1,19 +1,38 @@ #!/bin/bash - +APP_HOME="/workspaces/learning-bash-scripting-3212393/use-case" +SQL_FILE="sample.sql" START_DATE="2023-12-01" END_DATE="2023-12-18" -START_MONTH=$(date -d "$START_DATE" +%Y-%m) -END_MONTH=$(date -d "$END_DATE" +%Y-%m) +START_DATE=$(date -d $START_DATE +%Y-%m-%d ) +END_DATE=$(date -d $END_DATE +%Y-%m-%d ) +LOCAL_START_DATE=$(date -d $START_DATE +%Y-%m-%d ) + +START_MONTH=$(date -d $START_DATE +%Y-%m) +END_MONTH=$(date -d $END_DATE +%Y-%m) +LOCAL_START_MONTH=$(date -d $START_DATE +%Y-%m) if [ $START_MONTH == $END_MONTH ] then START_DATE=$START_DATE END_DATE=$END_DATE + VAR="'$START_DATE' and '$END_DATE'" + sed -i -e "s/\(date_key between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} else - START_DATE="null" - END_DATE="null" + echo null + while [ $LOCAL_START_MONTH != $END_MONTH ]; do + THE_START_DATE=$LOCAL_START_DATE + THE_END_DATE=$LOCAL_END_DATE + + VAR="'$THE_START_DATE' and '$THE_END_DATE'" + sed -i -e "s/\(date_key between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} + + LOCAL_START_DATE=$(date -d "$THE_START_DATE + 1 days" +%Y-%m-%d) + LOCAL_END_DATE= + done fi -echo $START_DATE -echo $END_DATE \ No newline at end of file +# echo $START_DATE +# echo $END_DATE +# echo $START_MONTH +# echo $END_MONTH \ No newline at end of file From b8a161d93e2beac0bf0f4793c7bb3ccc10db31c5 Mon Sep 17 00:00:00 2001 From: Sivakumar R <108018740+sivakumar-dr@users.noreply.github.com> Date: Thu, 21 Dec 2023 04:33:47 +0000 Subject: [PATCH 08/11] change directory name --- {use-case => use_case}/for-loops.sh | 0 {use-case => use_case}/sample.sql | 0 {use-case => use_case}/sed-test.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {use-case => use_case}/for-loops.sh (100%) rename {use-case => use_case}/sample.sql (100%) rename {use-case => use_case}/sed-test.sh (100%) diff --git a/use-case/for-loops.sh b/use_case/for-loops.sh similarity index 100% rename from use-case/for-loops.sh rename to use_case/for-loops.sh diff --git a/use-case/sample.sql b/use_case/sample.sql similarity index 100% rename from use-case/sample.sql rename to use_case/sample.sql diff --git a/use-case/sed-test.sh b/use_case/sed-test.sh similarity index 100% rename from use-case/sed-test.sh rename to use_case/sed-test.sh From c4d3aef9abeda9a158c1e9f8f832af5eef952b24 Mon Sep 17 00:00:00 2001 From: Sivakumar R <108018740+sivakumar-dr@users.noreply.github.com> Date: Fri, 22 Dec 2023 05:32:45 +0000 Subject: [PATCH 09/11] Remove file testing done --- use_case/date_test.sh | 36 +++++++++++++++++ use_case/rm_file.sh | 21 ++++++++++ use_case/sample.sql | 3 +- use_case/sed-test.sh | 39 +++++++++---------- ...bdp_backfill_logging_output-2023-12-21.txt | 2 + use_case/unix.txt | 2 + 6 files changed, 82 insertions(+), 21 deletions(-) create mode 100755 use_case/date_test.sh create mode 100755 use_case/rm_file.sh create mode 100644 use_case/transaction_count_bdp_backfill_logging_output-2023-12-21.txt create mode 100644 use_case/unix.txt diff --git a/use_case/date_test.sh b/use_case/date_test.sh new file mode 100755 index 0000000..9623338 --- /dev/null +++ b/use_case/date_test.sh @@ -0,0 +1,36 @@ +#!/bin/bash +APP_HOME="/workspaces/learning-bash-scripting-3212393/use-case" +SQL_FILE="sample.sql" +START_DATE="2023-01-01" +END_DATE="2023-12-18" + +START_DATE=$(date -d $START_DATE +%Y-%m-%d) +END_DATE=$(date -d $END_DATE +%Y-%m-%d) +LOCAL_START_DATE=$START_DATE + +START_MONTH=$(date -d $START_DATE +%Y-%m) +END_MONTH=$(date -d $END_DATE +%Y-%m) +LOCAL_START_MONTH=$START_MONTH + +echo $LOCAL_START_MONTH +echo $END_MONTH + +THE_START_DATE=$LOCAL_START_DATE +echo $THE_START_DATE + +THE_END_DATE=$(date -d "$(date -d "$THE_START_DATE +1 month" +%Y-%m-01) -1sec" +%F) +echo $THE_END_DATE + +THE_START_DATE=$(date -d "$THE_END_DATE +1 days" +%Y-%m-01) +echo $THE_START_DATE + +THE_END_DATE=$(date -d "$(date -d "$THE_START_DATE +1 month" +%Y-%m-01) -1sec" +%F) +echo $THE_END_DATE + +LOCAL_START_MONTH=$(date -d $THE_START_DATE +%Y-%m) +echo $LOCAL_START_MONTH +echo $END_MONTH + + + + diff --git a/use_case/rm_file.sh b/use_case/rm_file.sh new file mode 100755 index 0000000..e329be6 --- /dev/null +++ b/use_case/rm_file.sh @@ -0,0 +1,21 @@ +#!/bin/bash +APP_HOME=/workspaces/learning-bash-scripting-3212393 +SH_LOG_DIR=${APP_HOME}/use_case +LOG_FILE1="transaction_count_bdp_backfill_logging_output" + +RM_FILE="${SH_LOG_DIR}/${LOG_FILE1}-$(date +%Y-%m-%d).txt" + +RM_FILE_LINE_COUNT=$(head -n 3 -- "$RM_FILE" | wc -w) + +echo $RM_FILE_LINE_COUNT +if [[ ${RM_FILE_LINE_COUNT} -eq 0 ]] +then + rm -rf ${RM_FILE} + echo "${RM_FILE} empty log file removed" +elif [[ ${RM_FILE_LINE_COUNT} -gt 0 ]] +then + echo "${RM_FILE} log file is not empty" +elif [[ -f ${RM_FILE} ]] +then + echo "${RM_FILE} log file not found" +fi \ No newline at end of file diff --git a/use_case/sample.sql b/use_case/sample.sql index e7a08a7..09ff367 100644 --- a/use_case/sample.sql +++ b/use_case/sample.sql @@ -1,3 +1,4 @@ select * from employees -where date_key between '2023-12-01' and '2023-12-18' \ No newline at end of file +where date_key between '2023-12-01' and '2023-12-19' +where transaction_date_key between '2023-12-01' and '2023-12-19' \ No newline at end of file diff --git a/use_case/sed-test.sh b/use_case/sed-test.sh index 49906aa..e38a8e2 100755 --- a/use_case/sed-test.sh +++ b/use_case/sed-test.sh @@ -1,38 +1,37 @@ #!/bin/bash -APP_HOME="/workspaces/learning-bash-scripting-3212393/use-case" +APP_HOME="/workspaces/learning-bash-scripting-3212393/use_case" SQL_FILE="sample.sql" -START_DATE="2023-12-01" -END_DATE="2023-12-18" +START_DATE=${1} +END_DATE=${2} -START_DATE=$(date -d $START_DATE +%Y-%m-%d ) -END_DATE=$(date -d $END_DATE +%Y-%m-%d ) -LOCAL_START_DATE=$(date -d $START_DATE +%Y-%m-%d ) +START_DATE=$(date -d $START_DATE +%Y-%m-%d) +END_DATE=$(date -d $END_DATE +%Y-%m-%d) +LOCAL_START_DATE=$START_DATE START_MONTH=$(date -d $START_DATE +%Y-%m) END_MONTH=$(date -d $END_DATE +%Y-%m) -LOCAL_START_MONTH=$(date -d $START_DATE +%Y-%m) +LOCAL_START_MONTH=$START_MONTH if [ $START_MONTH == $END_MONTH ] then - START_DATE=$START_DATE - END_DATE=$END_DATE VAR="'$START_DATE' and '$END_DATE'" sed -i -e "s/\(date_key between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} + sed -i -e "s/\(transaction_date_key between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} else - echo null - while [ $LOCAL_START_MONTH != $END_MONTH ]; do + while [ $LOCAL_START_MONTH != $END_MONTH ] + do THE_START_DATE=$LOCAL_START_DATE - THE_END_DATE=$LOCAL_END_DATE + THE_END_DATE=$(date -d "$(date -d "$THE_START_DATE +1 month" +%Y-%m-01) -1sec" +%F) VAR="'$THE_START_DATE' and '$THE_END_DATE'" sed -i -e "s/\(date_key between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} + sed -i -e "s/\(transaction_date_key between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} - LOCAL_START_DATE=$(date -d "$THE_START_DATE + 1 days" +%Y-%m-%d) - LOCAL_END_DATE= + LOCAL_START_DATE=$(date -d "$THE_END_DATE +1 days" +%Y-%m-01) + LOCAL_START_MONTH=$(date -d $LOCAL_START_DATE +%Y-%m) done -fi - -# echo $START_DATE -# echo $END_DATE -# echo $START_MONTH -# echo $END_MONTH \ No newline at end of file + + VAR="'$LOCAL_START_DATE' and '$END_DATE'" + sed -i -e "s/\(date_key between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} + sed -i -e "s/\(transaction_date_key between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} +fi \ No newline at end of file diff --git a/use_case/transaction_count_bdp_backfill_logging_output-2023-12-21.txt b/use_case/transaction_count_bdp_backfill_logging_output-2023-12-21.txt new file mode 100644 index 0000000..76afcce --- /dev/null +++ b/use_case/transaction_count_bdp_backfill_logging_output-2023-12-21.txt @@ -0,0 +1,2 @@ +test +test \ No newline at end of file diff --git a/use_case/unix.txt b/use_case/unix.txt new file mode 100644 index 0000000..76afcce --- /dev/null +++ b/use_case/unix.txt @@ -0,0 +1,2 @@ +test +test \ No newline at end of file From 2550877f7f3de50c9ed93243b7c0df9f6af8bc8b Mon Sep 17 00:00:00 2001 From: Sivakumar R <108018740+sivakumar-dr@users.noreply.github.com> Date: Thu, 28 Dec 2023 14:55:13 +0000 Subject: [PATCH 10/11] ava data shell script tested --- use_case/ava_data.sh | 36 ++++++++++++++++++++++++++++++++++++ use_case/sample.sql | 4 ++-- 2 files changed, 38 insertions(+), 2 deletions(-) create mode 100755 use_case/ava_data.sh diff --git a/use_case/ava_data.sh b/use_case/ava_data.sh new file mode 100755 index 0000000..b996bc7 --- /dev/null +++ b/use_case/ava_data.sh @@ -0,0 +1,36 @@ +#!/bin/bash +app_home="/workspaces/learning-bash-scripting-3212393/use_case" +sql_file="sample.sql" +frequency=${1} +start_date=${2} +end_date=${3} + + + +if [[ -z ${frequency} ]] +then + echo "please provide the frequency" + exit 1 +fi + +if [[ ${frequency} == "custom" ]] && ([[ -z ${start_date} ]] || [[ -z ${end_date} ]]) +then + echo "Please provide the start_date and end_date" + exit 1 +fi + +if [[ ${frequency} == 'monthly' ]] +then + start_date=$(date -d "$(date -d "$($date) -1 month" +%Y-%m-01)" +%Y-%m-%d) + end_date=$(date -d "$(date -d "$start_date +1 month" +%Y-%m-01) -1sec" +%F) +fi + +if [[ ${FREQUENCY} == 'custom' ]] +then + start_date=$(date -d $start_date +%Y-%m-%d) + end_date=$(date -d $end_date +%Y-%m-%d) +fi + +var="'$start_date' and '$end_date'" +echo "Avalara ${FREQUENCY} recon for date_key between $var" +sed -i -e "s/\(date_key between \).*/\1$var/" ${app_home}/${sql_file} \ No newline at end of file diff --git a/use_case/sample.sql b/use_case/sample.sql index 09ff367..ff1ea48 100644 --- a/use_case/sample.sql +++ b/use_case/sample.sql @@ -1,4 +1,4 @@ select * from employees -where date_key between '2023-12-01' and '2023-12-19' -where transaction_date_key between '2023-12-01' and '2023-12-19' \ No newline at end of file +where date_key between '2023-12-01' and '2023-12-20' +where transaction_date_key between '2023-12-01' and '2023-12-20' \ No newline at end of file From f5e6917d8aa23657a66a003259e0adb8e3f3e76e Mon Sep 17 00:00:00 2001 From: Sivakumar R <108018740+sivakumar-dr@users.noreply.github.com> Date: Mon, 12 Feb 2024 12:21:21 +0000 Subject: [PATCH 11/11] sed test done for sql date changes --- use_case/sample.sql | 5 +++-- use_case/sed-test.sh | 12 ++++-------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/use_case/sample.sql b/use_case/sample.sql index ff1ea48..9094f30 100644 --- a/use_case/sample.sql +++ b/use_case/sample.sql @@ -1,4 +1,5 @@ select * from employees -where date_key between '2023-12-01' and '2023-12-20' -where transaction_date_key between '2023-12-01' and '2023-12-20' \ No newline at end of file +where date_key between '2024-01-01' and '2024-01-31' +where transaction_date_key between '2024-01-01' and '2024-01-31' +where cast(date_add(cast(n.reporting_date_key as date), -1) as varchar(15)) between '2024-01-01' and '2024-01-31' \ No newline at end of file diff --git a/use_case/sed-test.sh b/use_case/sed-test.sh index e38a8e2..8e60ed4 100755 --- a/use_case/sed-test.sh +++ b/use_case/sed-test.sh @@ -1,5 +1,5 @@ #!/bin/bash -APP_HOME="/workspaces/learning-bash-scripting-3212393/use_case" +APP_HOME="/workspaces/Linux-learning-bash-scripting/use_case" SQL_FILE="sample.sql" START_DATE=${1} END_DATE=${2} @@ -15,8 +15,7 @@ LOCAL_START_MONTH=$START_MONTH if [ $START_MONTH == $END_MONTH ] then VAR="'$START_DATE' and '$END_DATE'" - sed -i -e "s/\(date_key between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} - sed -i -e "s/\(transaction_date_key between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} + sed -i -e "s/\(between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} else while [ $LOCAL_START_MONTH != $END_MONTH ] do @@ -24,14 +23,11 @@ else THE_END_DATE=$(date -d "$(date -d "$THE_START_DATE +1 month" +%Y-%m-01) -1sec" +%F) VAR="'$THE_START_DATE' and '$THE_END_DATE'" - sed -i -e "s/\(date_key between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} - sed -i -e "s/\(transaction_date_key between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} - + sed -i -e "s/\(between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} LOCAL_START_DATE=$(date -d "$THE_END_DATE +1 days" +%Y-%m-01) LOCAL_START_MONTH=$(date -d $LOCAL_START_DATE +%Y-%m) done VAR="'$LOCAL_START_DATE' and '$END_DATE'" - sed -i -e "s/\(date_key between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} - sed -i -e "s/\(transaction_date_key between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} + sed -i -e "s/\(between \).*/\1$VAR/" ${APP_HOME}/${SQL_FILE} fi \ No newline at end of file