Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ism index patterns based on indexAlias/index #5117 #5118

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

ism index patterns based on indexAlias/index #5117 #5118

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d06fcb8
ism index patterns based on indexAlias/index #5117
kranthikirang Oct 26, 2024
6739a56
fixed integration tests with indexAlias
kranthikirang Nov 15, 2024
20b9749
change the policy test template to have dummy index-patterns
kranthikirang Nov 15, 2024
904e3ba
getIndexPattern test
kranthikirang Nov 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions ...n/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/AbstractIndexManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ public void setupIndex() throws IOException {
private void checkAndCreateIndexTemplate() throws IOException {
final boolean isISMEnabled = checkISMEnabled();
final Optional<String> policyIdOptional = isISMEnabled ?
ismPolicyManagementStrategy.checkAndCreatePolicy() :
ismPolicyManagementStrategy.checkAndCreatePolicy(configuredIndexAlias) :
kranthikirang marked this conversation as resolved.
Show resolved Hide resolved
Optional.empty();
if (!openSearchSinkConfiguration.getIndexConfiguration().getIndexTemplate().isEmpty()) {
checkAndCreateIndexTemplate(isISMEnabled, policyIdOptional.orElse(null));
Expand Down Expand Up @@ -258,8 +258,8 @@ final void checkAndCreateIndexTemplate(final boolean isISMEnabled, final String
templateStrategy.createTemplate(indexTemplate);
}

final Optional<String> checkAndCreatePolicy() throws IOException {
return ismPolicyManagementStrategy.checkAndCreatePolicy();
final Optional<String> checkAndCreatePolicy(final String indexAlias) throws IOException {
return ismPolicyManagementStrategy.checkAndCreatePolicy(indexAlias);
}

public void checkAndCreateIndex() throws IOException {
Expand Down Expand Up @@ -322,6 +322,7 @@ private void attachPolicy(
indexTemplate.putCustomSetting(IndexConstants.ISM_POLICY_ID_SETTING, ismPolicyId);
}
indexTemplate.putCustomSetting(IndexConstants.ISM_ROLLOVER_ALIAS_SETTING, rolloverAlias);
indexTemplate.putCustomSetting(IndexConstants.PLUGINS_ROLLOVER_ALIAS_SETTING, rolloverAlias);
}

}
1 change: 1 addition & 0 deletions ...rc/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ public class IndexConstants {
public static final String ISM_ENABLED_SETTING = "opendistro.index_state_management.enabled";
public static final String ISM_POLICY_ID_SETTING = "opendistro.index_state_management.policy_id";
public static final String ISM_ROLLOVER_ALIAS_SETTING = "opendistro.index_state_management.rollover_alias";
public static final String PLUGINS_ROLLOVER_ALIAS_SETTING = "plugins.index_state_management.rollover_alias";
// TODO: extract out version number into version enum
public static final String SERVICE_MAP_DEFAULT_TEMPLATE_FILE = "otel-v1-apm-service-map-index-template.json";

Expand Down
12 changes: 11 additions & 1 deletion ...in/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IsmPolicyManagement.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import io.micrometer.core.instrument.util.StringUtils;
import org.opensearch.client.Request;
import org.opensearch.client.ResponseException;
Expand Down Expand Up @@ -93,10 +94,19 @@ public IsmPolicyManagement(final OpenSearchClient openSearchClient,
}

@Override
public Optional<String> checkAndCreatePolicy() throws IOException {
public Optional<String> checkAndCreatePolicy(final String indexAlias) throws IOException {
final String policyManagementEndpoint = POLICY_MANAGEMENT_ENDPOINT + policyName;

String policyJsonString = retrievePolicyJsonString(policyFile);
if(!indexAlias.isEmpty()) {
kranthikirang marked this conversation as resolved.
Show resolved Hide resolved
final ObjectMapper mapper = new ObjectMapper();
final JsonNode jsonNode = mapper.readTree(policyJsonString);
final ArrayNode iparray = mapper.createArrayNode();
iparray.add(indexAlias + "*");
((ObjectNode) jsonNode.get("policy").get("ism_template")).put("index_patterns", iparray);
policyJsonString = jsonNode.toString();
}
LOG.debug("Got the policystring as {} and indexAlias as {}", policyJsonString, indexAlias);
Request request = createPolicyRequestFromFile(policyManagementEndpoint, policyJsonString);

try {
Expand Down
2 changes: 1 addition & 1 deletion ...org/opensearch/dataprepper/plugins/sink/opensearch/index/IsmPolicyManagementStrategy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

interface IsmPolicyManagementStrategy {

Optional<String> checkAndCreatePolicy() throws IOException;
Optional<String> checkAndCreatePolicy(final String indexAlias) throws IOException;
kranthikirang marked this conversation as resolved.
Show resolved Hide resolved

List<String> getIndexPatterns(final String indexAlias);

Expand Down
2 changes: 1 addition & 1 deletion .../java/org/opensearch/dataprepper/plugins/sink/opensearch/index/NoIsmPolicyManagement.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public NoIsmPolicyManagement(final OpenSearchClient openSearchClient,
}

@Override
public Optional<String> checkAndCreatePolicy() throws IOException {
public Optional<String> checkAndCreatePolicy(final String indexAlias) throws IOException {
return Optional.empty();
}

Expand Down
Loading