Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add ES7 support for ES6->7 upgrade path #2

Merged
merged 1 commit into from
May 23, 2022
Merged

Add ES7 support for ES6->7 upgrade path #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions cli/app.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,10 @@ func Reindex() cli.Command {
cli.BoolFlag{Name: "bulk-indexing", Usage: "set refresh_interval to -1 and set number_of_replicas to 0 when reindexing and revert afterwards."},
cli.BoolFlag{Name: "version-external", Usage: "set version_type to external. This will only index documents if they don't exist or the source doc is at a higher version"},
cli.BoolFlag{Name: "no-update-alias", Usage: "don't update the index alias. This setting will also not revert the refresh_interval and number_of_replicas if bulk-indexing is set"},
cli.BoolFlag{Name: "include-type-name", Usage: "passes 'include_type_name=true' for put index template request. Used for ES6->7 upgrade. https://www.elastic.co/blog/moving-from-types-to-typeless-apis-in-elasticsearch-7-0"},
cli.StringFlag{Name: "reindex-host-allocation", Usage: "Optional target host for the reindex to happen on. eg. 'es-reindex-*'"},
cli.StringFlag{Name: "dest-host-allocation", Usage: "Optional target host once the reindex is complete. eg. 'es-data-*'"},
cli.StringFlag{Name: "extra-suffix", Usage: "Optional extra suffix name to add to index name (after date). Ignored if dest-index is set"},
},
Action: func(c *cli.Context) error {
if c.NArg() == 0 || c.NArg() > 1 {
Expand All @@ -157,7 +159,7 @@ func Reindex() cli.Command {
// Single index
filePath := c.Args().First()

newIndex, err := elasticsearch.UpdateTemplateAndCreateNewIndex(client, filePath, c.String("dest-index"), c.Bool("bulk-indexing"))
newIndex, err := elasticsearch.UpdateTemplateAndCreateNewIndex(client, filePath, c.String("dest-index"), c.Bool("bulk-indexing"), c.Bool("include-type-name"), c.String("extra-suffix"))
if err != nil {
return err
}
Expand Down Expand Up @@ -193,7 +195,7 @@ func Reindex() cli.Command {
}

directory := c.Args().First()
aliasToNewIndex, err := elasticsearch.UpdateTemplatesAndCreateNewIndices(client, directory, c.Bool("bulk-indexing"))
aliasToNewIndex, err := elasticsearch.UpdateTemplatesAndCreateNewIndices(client, directory, c.Bool("bulk-indexing"), c.Bool("include-type-name"), c.String("extra-suffix"))
if err != nil {
return err
}
Expand Down
13 changes: 8 additions & 5 deletions elasticsearch/elasdx.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ var Updated = color.GreenString("Updated ")
var Reindexed = color.YellowString("Reindexed ")
var Info = color.YellowString("Info ")

func UpdateTemplateAndCreateNewIndex(client *elastic.Client, filePath, newIndexName string, bulkIndexing bool) (string, error) {
func UpdateTemplateAndCreateNewIndex(client *elastic.Client, filePath, newIndexName string, bulkIndexing bool, includeTypeName bool, extraSuffix string) (string, error) {
bytes, err := ioutil.ReadFile(filePath)
if err != nil {
return "", errors.Wrapf(err, "failed reading file %s", filePath)
Expand All @@ -39,7 +39,7 @@ func UpdateTemplateAndCreateNewIndex(client *elastic.Client, filePath, newIndexN
mapping := string(bytes)

// Update the template
indexPutTemplate, err := client.IndexPutTemplate(index).BodyString(mapping).Do(context.Background())
indexPutTemplate, err := client.IndexPutTemplate(index).IncludeTypeName(includeTypeName).BodyString(mapping).Do(context.Background())
if err != nil {
return "", errors.Wrapf(err, "failed updating index template %s", index)
}
Expand All @@ -50,8 +50,11 @@ func UpdateTemplateAndCreateNewIndex(client *elastic.Client, filePath, newIndexN
fmt.Printf("%s %s %s\n", IndexTemplatePrefix, Updated, index)

// Create a unique time-stamped index
dateSuffix := time.Now().Format("2006-01-02-15:04:05")
dateSuffix := strings.ReplaceAll(time.Now().Format("2006-01-02-15:04:05"), ":", "-")
if newIndexName == "" {
if extraSuffix != "" {
dateSuffix = fmt.Sprintf("%s-%s", dateSuffix, extraSuffix)
}
newIndexName = fmt.Sprintf("%s-%s", index, dateSuffix)
}

Expand Down Expand Up @@ -95,7 +98,7 @@ func UpdateTemplateAndCreateNewIndex(client *elastic.Client, filePath, newIndexN
return newIndexName, nil
}

func UpdateTemplatesAndCreateNewIndices(client *elastic.Client, templatesDir string, bulkIndexing bool) (map[string]string, error) {
func UpdateTemplatesAndCreateNewIndices(client *elastic.Client, templatesDir string, bulkIndexing bool, includeTypeName bool, extraSuffix string) (map[string]string, error) {
files, err := ioutil.ReadDir(templatesDir)
if err != nil {
return nil, errors.Wrapf(err, "failed reading files in directory %s", templatesDir)
Expand All @@ -110,7 +113,7 @@ func UpdateTemplatesAndCreateNewIndices(client *elastic.Client, templatesDir str
}

filePath := filepath.Join(templatesDir, file.Name())
newIndex, err := UpdateTemplateAndCreateNewIndex(client, filePath, "", bulkIndexing)
newIndex, err := UpdateTemplateAndCreateNewIndex(client, filePath, "", bulkIndexing, includeTypeName, extraSuffix)
if err != nil {
return nil, errors.Wrapf(err, "failed create new index from updated template %s", filePath)
}
Expand Down
6 changes: 3 additions & 3 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ require (
github.com/fatih/color v1.7.0
github.com/fortytw2/leaktest v1.3.0 // indirect
github.com/google/go-cmp v0.3.0 // indirect
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.2 // indirect
github.com/olivere/elastic v6.2.19+incompatible
github.com/pkg/errors v0.8.1
github.com/olivere/elastic v6.2.37+incompatible
github.com/pkg/errors v0.9.1
github.com/urfave/cli v1.20.0
)
8 changes: 8 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,24 @@ github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/olivere/elastic v6.2.19+incompatible h1:FYiohzobKD2tib3sHgEHNBSzcw1onqApaumIBBstchQ=
github.com/olivere/elastic v6.2.19+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
github.com/olivere/elastic v6.2.37+incompatible h1:UfSGJem5czY+x/LqxgeCBgjDn6St+z8OnsCuxwD3L0U=
github.com/olivere/elastic v6.2.37+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
Expand Down