Skip to content

Port the remaining old indices search tests to full cluster restart qa module #25275

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
martijnvg merged 1 commit into from
Jun 19, 2017
Merged

Port the remaining old indices search tests to full cluster restart qa module #25275

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
184 changes: 0 additions & 184 deletions core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,45 +19,23 @@

package org.elasticsearch.bwcompat;

import org.apache.lucene.search.Explanation;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.elasticsearch.Version;
import org.elasticsearch.VersionTests;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
import org.elasticsearch.client.Requests;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.gateway.MetaDataStateFormat;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.node.Node;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHitField;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.InternalSettingsPlugin;
import org.elasticsearch.test.OldIndexUtils;
import org.elasticsearch.test.VersionUtils;
import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
import org.hamcrest.Matchers;
import org.junit.AfterClass;
import org.junit.Before;

Expand All @@ -70,16 +48,10 @@
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.SortedSet;
import java.util.TreeSet;

import static org.elasticsearch.index.query.QueryBuilders.matchPhraseQuery;
import static org.elasticsearch.test.OldIndexUtils.assertUpgradeWorks;
import static org.elasticsearch.test.OldIndexUtils.getIndexDir;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;

// needs at least 2 nodes since it bumps replicas to 1
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
Expand Down Expand Up @@ -120,59 +92,6 @@ public Settings nodeSettings(int ord) {
return OldIndexUtils.getSettings();
}

void setupCluster() throws Exception {
List<String> replicas = internalCluster().startNodes(1); // for replicas

Path baseTempDir = createTempDir();
// start single data path node
Settings.Builder nodeSettings = Settings.builder()
.put(Environment.PATH_DATA_SETTING.getKey(), baseTempDir.resolve("single-path").toAbsolutePath())
.put(Node.NODE_MASTER_SETTING.getKey(), false); // workaround for dangling index loading issue when node is master
singleDataPathNodeName = internalCluster().startNode(nodeSettings);

// start multi data path node
nodeSettings = Settings.builder()
.put(Environment.PATH_DATA_SETTING.getKey(), baseTempDir.resolve("multi-path1").toAbsolutePath() + "," + baseTempDir
.resolve("multi-path2").toAbsolutePath())
.put(Node.NODE_MASTER_SETTING.getKey(), false); // workaround for dangling index loading issue when node is master
multiDataPathNodeName = internalCluster().startNode(nodeSettings);

// find single data path dir
Path[] nodePaths = internalCluster().getInstance(NodeEnvironment.class, singleDataPathNodeName).nodeDataPaths();
assertEquals(1, nodePaths.length);
singleDataPath = nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER);
assertFalse(Files.exists(singleDataPath));
Files.createDirectories(singleDataPath);
logger.info("--> Single data path: {}", singleDataPath);

// find multi data path dirs
nodePaths = internalCluster().getInstance(NodeEnvironment.class, multiDataPathNodeName).nodeDataPaths();
assertEquals(2, nodePaths.length);
multiDataPath = new Path[]{nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER),
nodePaths[1].resolve(NodeEnvironment.INDICES_FOLDER)};
assertFalse(Files.exists(multiDataPath[0]));
assertFalse(Files.exists(multiDataPath[1]));
Files.createDirectories(multiDataPath[0]);
Files.createDirectories(multiDataPath[1]);
logger.info("--> Multi data paths: {}, {}", multiDataPath[0], multiDataPath[1]);
ensureGreen();
}

void upgradeIndexFolder() throws Exception {
OldIndexUtils.upgradeIndexFolder(internalCluster(), singleDataPathNodeName);
OldIndexUtils.upgradeIndexFolder(internalCluster(), multiDataPathNodeName);
}

void importIndex(String indexName) throws IOException {
// force reloading dangling indices with a cluster state republish
client().admin().cluster().prepareReroute().get();
ensureGreen(indexName);
}

void unloadIndex(String indexName) throws Exception {
assertAcked(client().admin().indices().prepareDelete(indexName).get());
}

public void testAllVersionsTested() throws Exception {
SortedSet<String> expectedVersions = new TreeSet<>();
for (Version v : VersionUtils.allReleasedVersions()) {
Expand All @@ -198,115 +117,12 @@ public void testAllVersionsTested() throws Exception {
}
}

public void testOldIndexes() throws Exception {
setupCluster();

Collections.shuffle(indexes, random());
for (String index : indexes) {
long startTime = System.currentTimeMillis();
logger.info("--> Testing old index {}", index);
assertOldIndexWorks(index);
logger.info("--> Done testing {}, took {} seconds", index, (System.currentTimeMillis() - startTime) / 1000.0);
}
}

void assertOldIndexWorks(String index) throws Exception {
Version version = OldIndexUtils.extractVersion(index);
Path[] paths;
if (randomBoolean()) {
logger.info("--> injecting index [{}] into single data path", index);
paths = new Path[]{singleDataPath};
} else {
logger.info("--> injecting index [{}] into multi data path", index);
paths = multiDataPath;
}

String indexName = index.replace(".zip", "").toLowerCase(Locale.ROOT).replace("unsupported-", "index-");
OldIndexUtils.loadIndex(indexName, index, createTempDir(), getBwcIndicesPath(), logger, paths);
// we explicitly upgrade the index folders as these indices
// are imported as dangling indices and not available on
// node startup
upgradeIndexFolder();
importIndex(indexName);
assertUpgradeWorks(client(), indexName, version);
assertPositionIncrementGapDefaults(indexName, version);
assertAliasWithBadName(indexName, version);
assertStoredBinaryFields(indexName, version);
unloadIndex(indexName);
}

void assertPositionIncrementGapDefaults(String indexName, Version version) throws Exception {
client().prepareIndex(indexName, "doc", "position_gap_test").setSource("string", Arrays.asList("one", "two three"))
.setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();

// Baseline - phrase query finds matches in the same field value
assertHitCount(client().prepareSearch(indexName).setQuery(matchPhraseQuery("string", "two three")).get(), 1);

// No match across gaps when slop < position gap
assertHitCount(
client().prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two").slop(99)).get(),
0);

// Match across gaps when slop >= position gap
assertHitCount(client().prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two").slop(100)).get(), 1);
assertHitCount(client().prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two").slop(101)).get(),
1);

// No match across gap using default slop with default positionIncrementGap
assertHitCount(client().prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two")).get(), 0);

// Nor with small-ish values
assertHitCount(client().prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two").slop(5)).get(), 0);
assertHitCount(client().prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two").slop(50)).get(), 0);

// But huge-ish values still match
assertHitCount(client().prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two").slop(500)).get(), 1);
}

private static final Version VERSION_5_1_0_UNRELEASED = Version.fromString("5.1.0");

public void testUnreleasedVersion() {
VersionTests.assertUnknownVersion(VERSION_5_1_0_UNRELEASED);
}

/**
* Search on an alias that contains illegal characters that would prevent it from being created after 5.1.0. It should still be
* search-able though.
*/
void assertAliasWithBadName(String indexName, Version version) throws Exception {
if (version.onOrAfter(VERSION_5_1_0_UNRELEASED)) {
return;
}
// We can read from the alias just like we can read from the index.
String aliasName = "#" + indexName;
long totalDocs = client().prepareSearch(indexName).setSize(0).get().getHits().getTotalHits();
assertHitCount(client().prepareSearch(aliasName).setSize(0).get(), totalDocs);
assertThat(totalDocs, greaterThanOrEqualTo(2000L));

// We can remove the alias.
assertAcked(client().admin().indices().prepareAliases().removeAlias(indexName, aliasName).get());
assertFalse(client().admin().indices().prepareAliasesExist(aliasName).get().exists());
}

/**
* Make sure we can load stored binary fields.
*/
void assertStoredBinaryFields(String indexName, Version version) throws Exception {
SearchRequestBuilder builder = client().prepareSearch(indexName);
builder.setQuery(QueryBuilders.matchAllQuery());
builder.setSize(100);
builder.addStoredField("binary");
SearchHits hits = builder.get().getHits();
assertEquals(100, hits.getHits().length);
for(SearchHit hit : hits) {
SearchHitField field = hit.field("binary");
assertNotNull(field);
Object value = field.getValue();
assertTrue(value instanceof BytesArray);
assertEquals(16, ((BytesArray) value).length());
}
}

private Path getNodeDir(String indexFile) throws IOException {
Path unzipDir = createTempDir();
Path unzipDataDir = unzipDir.resolve("data");
Expand Down
2 changes: 1 addition & 1 deletion qa/full-cluster-restart/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ for (Version version : indexCompatVersions) {
dependsOn = [upgradedClusterTest]
}

if (project.bwc_tests_enabled == false) {
if (project.bwc_tests_enabled) {
bwcTest.dependsOn(versionBwcTest)
}
}
Expand Down
Loading