From bbf70983779a01553ce1ee027f2f7c59df44369d Mon Sep 17 00:00:00 2001 From: Gonchik Tsymzhitov Date: Tue, 28 Jan 2025 11:48:37 +0200 Subject: [PATCH] Just read docs and adjust small type --- docker/playground/filldb/filldb.sh | 2 +- docs/architecture.md | 7 ++++++- docs/built_in_transformers/dynamic_parameters.md | 2 +- .../random_amount_with_currency.md | 2 +- .../standard_transformers/random_day_of_week.md | 2 +- .../standard_transformers/random_timezone.md | 2 +- .../standard_transformers/random_url.md | 2 +- docs/commands/validate.md | 2 +- docs/configuration.md | 2 +- docs/database_subset.md | 2 +- docs/installation.md | 2 +- docs/release_notes/greenmask_0_1_5.md | 2 +- internal/db/postgres/dumpers/transformation_window.go | 2 +- .../db/postgres/transformers/custom/custom_cmd.go | 4 ++-- pkg/toolkit/expr.go | 10 +++++----- pkg/toolkit/template_functions.go | 11 ++++------- 16 files changed, 29 insertions(+), 27 deletions(-) diff --git a/docker/playground/filldb/filldb.sh b/docker/playground/filldb/filldb.sh index bdfbeefa..adb84429 100644 --- a/docker/playground/filldb/filldb.sh +++ b/docker/playground/filldb/filldb.sh @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -cd $TMP_DIR/postgresDBSamples/adventureworks +cd $TMP_DIR/postgresDBSamples/adventureworks || exit if ! psql -lqt -p 5432 -h playground-db -U postgres | cut -d \| -f 1 | grep -qw $ORIGINAL_DB_NAME; then psql -p 5432 -h playground-db -U postgres -c "CREATE DATABASE $ORIGINAL_DB_NAME;" diff --git a/docs/architecture.md b/docs/architecture.md index 459ac5aa..0b2b59c4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,7 +2,12 @@ ## Introduction -It is evident that the most appropriate approach for executing logical backup dumping and restoration is by leveraging the core PostgreSQL utilities, specifically `pg_dump` and `pg_restore`. Greenmask has been purposefully designed to align with PostgreSQL's native utilities, ensuring compatibility. Greenmask primarily handles data dumping operations independently and delegates the responsibilities of schema dumping and restoration to `pg_dump` and `pg_restore` respectively, maintaining seamless integration with PostgreSQL's standard tools. +It is evident that the most appropriate approach for executing logical backup dumping +and restoration is by leveraging the core PostgreSQL utilities, specifically `pg_dump` and `pg_restore`. +Greenmask has been purposefully designed to align with PostgreSQL's native utilities, ensuring compatibility. +Greenmask primarily handles data dumping operations independently and delegates +the responsibilities of schema dumping and restoration to `pg_dump` and `pg_restore` respectively, +maintaining seamless integration with PostgreSQL's standard tools. ## Backup process diff --git a/docs/built_in_transformers/dynamic_parameters.md b/docs/built_in_transformers/dynamic_parameters.md index a1a98da1..807f97fc 100644 --- a/docs/built_in_transformers/dynamic_parameters.md +++ b/docs/built_in_transformers/dynamic_parameters.md @@ -117,7 +117,7 @@ constraints, you can use dynamic parameters in the `RandomDate` transformer: template: '{{ .GetValue | tsModify "18 years" | .EncodeValue }}' # (7) ``` -1. Firstly we generate the `RadnomDate` for birthdate column. The result of the transformation will used as the minimum +1. Firstly we generate the `RadnomDate` for birthdate column. The result of the transformation will use as the minimum value for the next transformation for `hiredate` column. 2. Apply the template for static parameter. It calculates the now date and subtracts `30` years from it. The result is `1994`. The function tsModify return not a raw data, but time.Time object. For getting the raw value suitable for diff --git a/docs/built_in_transformers/standard_transformers/random_amount_with_currency.md b/docs/built_in_transformers/standard_transformers/random_amount_with_currency.md index a70d3d8c..e50e8d2d 100644 --- a/docs/built_in_transformers/standard_transformers/random_amount_with_currency.md +++ b/docs/built_in_transformers/standard_transformers/random_amount_with_currency.md @@ -9,7 +9,7 @@ The `RandomAmountWithCurrency` transformer is specifically designed to populate ## Description -This transformer automatically generates random financial amounts along with corresponding global currency codes (e. g., `250.00 USD`, `300.00 EUR`), injecting them into the designated database column. It provides a straightforward solution for populating financial records with varied and realistic data, suitable for testing payment systems, data anonymization, and simulation of economic models. +This transformer automatically generates random financial amounts along with corresponding global currency codes (e.g., `250.00 USD`, `300.00 EUR`), injecting them into the designated database column. It provides a straightforward solution for populating financial records with varied and realistic data, suitable for testing payment systems, data anonymization, and simulation of economic models. ## Example: Populate the `payments` table with random amounts and currencies diff --git a/docs/built_in_transformers/standard_transformers/random_day_of_week.md b/docs/built_in_transformers/standard_transformers/random_day_of_week.md index 3caae1ee..b5b8b3a8 100644 --- a/docs/built_in_transformers/standard_transformers/random_day_of_week.md +++ b/docs/built_in_transformers/standard_transformers/random_day_of_week.md @@ -9,7 +9,7 @@ The `RandomDayOfWeek` transformer is specifically designed to fill specified dat ## Description -Utilizing the `faker` library, the `RandomDayOfWeek` transformer generates names of days (e. g., Monday, Tuesday) at random. This transformer can be applied to any text or varchar column in a database, introducing variability and realism into data sets that need to represent days of the week in a non-specific manner. +Utilizing the `faker` library, the `RandomDayOfWeek` transformer generates names of days (e.g., Monday, Tuesday) at random. This transformer can be applied to any text or varchar column in a database, introducing variability and realism into data sets that need to represent days of the week in a non-specific manner. ## Example: Populate random days of the week for the `work_schedule` table diff --git a/docs/built_in_transformers/standard_transformers/random_timezone.md b/docs/built_in_transformers/standard_transformers/random_timezone.md index e5456c37..98c19744 100644 --- a/docs/built_in_transformers/standard_transformers/random_timezone.md +++ b/docs/built_in_transformers/standard_transformers/random_timezone.md @@ -9,7 +9,7 @@ The `RandomTimezone` transformer is designed to populate specified database colu ## Description -Utilizing a comprehensive library or algorithm for generating timezone data, the `RandomTimezone` transformer provides random timezone strings (e. g., "America/New_York", "Europe/London") for database columns. This feature enables the creation of diverse and realistic datasets by simulating timezone information for user profiles, event timings, or any other data requiring timezone context. +Utilizing a comprehensive library or algorithm for generating timezone data, the `RandomTimezone` transformer provides random timezone strings (e.g., "America/New_York", "Europe/London") for database columns. This feature enables the creation of diverse and realistic datasets by simulating timezone information for user profiles, event timings, or any other data requiring timezone context. ## Example: Populate random timezone strings for the `user_accounts` table diff --git a/docs/built_in_transformers/standard_transformers/random_url.md b/docs/built_in_transformers/standard_transformers/random_url.md index a48d38b7..6823a128 100644 --- a/docs/built_in_transformers/standard_transformers/random_url.md +++ b/docs/built_in_transformers/standard_transformers/random_url.md @@ -9,7 +9,7 @@ The `RandomURL` transformer is designed to populate specified database columns w ## Description -Utilizing advanced algorithms or libraries for generating URL strings, the `RandomURL` transformer injects random, plausible URLs into the designated database column. Each generated URL is structured to include the protocol (e. g., "http://", "https://"), domain name, and path, offering a realistic range of web addresses for various applications. +Utilizing advanced algorithms or libraries for generating URL strings, the `RandomURL` transformer injects random, plausible URLs into the designated database column. Each generated URL is structured to include the protocol (e.g., "http://", "https://"), domain name, and path, offering a realistic range of web addresses for various applications. ## Example: Populate random URLs for the `webpages` table diff --git a/docs/commands/validate.md b/docs/commands/validate.md index 26692286..5681d833 100644 --- a/docs/commands/validate.md +++ b/docs/commands/validate.md @@ -30,7 +30,7 @@ All of those cases may be used for CI/CD pipelines to stop the process when some useful when `--schema` flag is used - this allows to avoid data leakage when schema changed. You can use the `--table` flag multiple times to specify the tables you want to check. Tables can be written with -or without schema names (e. g., `public.table_name` or `table_name`). If you specify multiple tables from different +or without schema names (e.g., `public.table_name` or `table_name`). If you specify multiple tables from different schemas, an error will be thrown. To start validation, use the following command: diff --git a/docs/configuration.md b/docs/configuration.md index e88b4246..a8a521b9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -223,7 +223,7 @@ validate: ``` { .annotate } -1. A list of tables to validate. If this list is not empty, the validation operation will only be performed for the specified tables. Tables can be written with or without the schema name (e. g., `"public.cart"` or `"orders"`). +1. A list of tables to validate. If this list is not empty, the validation operation will only be performed for the specified tables. Tables can be written with or without the schema name (e.g., `"public.cart"` or `"orders"`). 2. Specifies whether to perform data transformation for a limited set of rows. If set to `true`, data transformation will be performed, and the number of rows transformed will be limited to the value specified in the `rows_limit` parameter (default is `10`). 3. Specifies whether to perform diff operations for the transformed data. If set to `true`, the validation process will **find the differences between the original and transformed data**. See more details in the [validate command documentation](commands/validate.md). 4. Limits the number of rows to be transformed during validation. The default limit is `10` rows, but you can change it by modifying this parameter. diff --git a/docs/database_subset.md b/docs/database_subset.md index 5ee2bf02..907e5a37 100644 --- a/docs/database_subset.md +++ b/docs/database_subset.md @@ -117,7 +117,7 @@ dump: Greenmask supports polymorphic references. You can define a virtual reference for a table with polymorphic references using `polymorphic_exprs` attribute. The `polymorphic_exprs` attribute is a list of expressions that are used to make -a polymorphic reference. For instance we might have a table `comments` that has polymorphic reference to `posts` and +a polymorphic reference. For instance, we might have a table `comments` that has polymorphic reference to `posts` and `videos`. The table comments might have `commentable_id` and `commentable_type` columns. The `commentable_type` column contains the type of the table that is referenced by the `commentable_id` column. The example of the config: diff --git a/docs/installation.md b/docs/installation.md index 29b64cdd..331144d1 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -20,7 +20,7 @@ docker run -it greenmask/greenmask:latest 2. GitHub container registry -To run the greenmask container from Github registry, use the following command: +To run the greenmask container from GitHub registry, use the following command: ```shell docker run -it ghcr.io/greenmaskio/greenmask:latest ``` diff --git a/docs/release_notes/greenmask_0_1_5.md b/docs/release_notes/greenmask_0_1_5.md index f9c04cc3..9212e8fc 100644 --- a/docs/release_notes/greenmask_0_1_5.md +++ b/docs/release_notes/greenmask_0_1_5.md @@ -8,7 +8,7 @@ Added a new Greenmask CLI command—[show-transformer](../commands/show-transfor ## Improvements -- The [Hash transformer](../built_in_transformers/standard_transformers/hash.md) has been completely remastered and now has the `function` parameter to choose from several hash algorithm options and the `max_length` parameter to truncate the hash tail. +- The [Hash transformer](../built_in_transformers/standard_transformers/hash.md) has been completely remastered and now has the `function` parameter to choose from some hash algorithm options and the `max_length` parameter to truncate the hash tail. - Split information about transformers between the `list-transformers` and new `show-transformer` CLI commands, which allows for more comprehensible and useful outputs for both commands - Added error severity for the `Cmd` parameter validator - Improved UX for the Greenmask release binaries diff --git a/internal/db/postgres/dumpers/transformation_window.go b/internal/db/postgres/dumpers/transformation_window.go index 2fd72b10..b0e1b5de 100644 --- a/internal/db/postgres/dumpers/transformation_window.go +++ b/internal/db/postgres/dumpers/transformation_window.go @@ -111,7 +111,7 @@ func (tw *transformationWindow) close() { close(tw.done) } -// Transform - runs the transformation for the record in the window. This function checks the when +// Transform - runs the transformation for the record in the window. This function checks when // condition of the transformer and if true sends a signal to the transformer goroutine to run the transformation func (tw *transformationWindow) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error) { tw.r = r diff --git a/internal/db/postgres/transformers/custom/custom_cmd.go b/internal/db/postgres/transformers/custom/custom_cmd.go index 30943dbb..4faec38d 100644 --- a/internal/db/postgres/transformers/custom/custom_cmd.go +++ b/internal/db/postgres/transformers/custom/custom_cmd.go @@ -307,8 +307,8 @@ func (ct *CmdTransformer) stderrForwarder(ctx context.Context) error { lineNum := 0 // This is required for convenient verbosity of output. // Write "stderr forwarding" log message each 500ms otherwise just print received stderr data - // If it does not use this logic each line would be covered with "stderr forwarding" message and it will be - // complicated to recognize the traceback or multiline message + // If it does not use this logic each line would be covered with "stderr forwarding" message, + // and it will be complicated to recognize the traceback or multiline message for { line, _, err := ct.StderrReader.ReadLine() if err != nil { diff --git a/pkg/toolkit/expr.go b/pkg/toolkit/expr.go index 8ead7a4f..9e7888cf 100644 --- a/pkg/toolkit/expr.go +++ b/pkg/toolkit/expr.go @@ -1,4 +1,4 @@ -// An expression handler for the toolkit package. It is used to evaluate the when condition of the record. +// An expression handler for the toolkit package. It is used to evaluate when condition of the record. // Might be used in transformation conditions and other places where the record is used. package toolkit @@ -26,9 +26,9 @@ type WhenCond struct { env map[string]any } -// NewWhenCond - creates a new WhenCond object. It compiles the when condition and returns the compiled program +// NewWhenCond - creates a new WhenCond object. It compiles when condition and returns the compiled program // and the record context with the functions for the columns. The functions represent the column names and return the -// column values. If the when condition is empty, the WhenCond object will always return true. +// column values. If when condition is empty, the WhenCond object will always return true. func NewWhenCond(when string, driver *Driver, meta map[string]any) (*WhenCond, ValidationWarnings) { var ( rc *RecordContext @@ -51,7 +51,7 @@ func NewWhenCond(when string, driver *Driver, meta map[string]any) (*WhenCond, V }, nil } -// Evaluate - evaluates the when condition. If the when condition is empty, it will always return true. +// Evaluate - evaluates when condition. If when condition is empty, it will always return true. func (wc *WhenCond) Evaluate(r *Record) (bool, error) { if wc.whenCond == nil { return true, nil @@ -71,7 +71,7 @@ func (wc *WhenCond) Evaluate(r *Record) (bool, error) { return false, fmt.Errorf("when condition should return boolean, got (%T) and value %+v", cond, cond) } -// compileCond compiles the when condition and returns the compiled program and the record context +// compileCond compiles when condition and returns the compiled program and the record context // with the functions for the columns. The functions represent the column names and return the column values. // meta - additional meta information for debugging the compilation process func compileCond(whenCond string, driver *Driver, meta map[string]any) ( diff --git a/pkg/toolkit/template_functions.go b/pkg/toolkit/template_functions.go index bc073763..cce4b6f5 100644 --- a/pkg/toolkit/template_functions.go +++ b/pkg/toolkit/template_functions.go @@ -23,10 +23,6 @@ import ( "text/template" "time" - "github.com/Masterminds/sprig/v3" - "github.com/ggwhite/go-masker" - "github.com/go-faker/faker/v4" - "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/spf13/cast" "github.com/tidwall/gjson" @@ -459,7 +455,8 @@ func randomDate(randGen *rand.Rand, min, max time.Time) (time.Time, error) { return *(RandomDate(randGen, &min, &max)), nil } -// randomFloat - generate float randomly in the interval [min, max] with precision. By default precision is 4 digits +// randomFloat - generate float randomly in the interval [min, max] with precision. +// By default, precision is 4 digits func randomFloat(randGen *rand.Rand, min, max any, decimal ...any) (float64, error) { var err error var p = 4 @@ -503,8 +500,8 @@ func randomInt(randGen *rand.Rand, min, max any) (int64, error) { return RandomInt(randGen, minInt, maxInt), nil } -// randomString - generate random string in the provided min and max length using provided symbols. By default symbols -// are "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890" +// randomString - generate random string in the provided min and max length using provided symbols. +// By default, symbols are "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890" func randomString(randGen *rand.Rand, minLength, maxLength any, symbols ...string) (string, error) { s := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890") if len(symbols) > 0 {