Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

✨ Source MongoDB V2: Handle statistics collection failure gracefully #28692

Merged
merged 6 commits into from
Jul 25, 2023
Merged

✨ Source MongoDB V2: Handle statistics collection failure gracefully #28692

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4334703
Fix bug introduced by statistics collection
jdpgrailsdev Jul 25, 2023
034fde1
Fix syntax
jdpgrailsdev Jul 25, 2023
66fffa5
Formatting
jdpgrailsdev Jul 25, 2023
350f49b
Fix failing test
jdpgrailsdev Jul 25, 2023
f951606
Fix failing test
jdpgrailsdev Jul 25, 2023
4912b15
Update release notes
jdpgrailsdev Jul 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoDatabase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.annotations.VisibleForTesting;
import com.mongodb.ConnectionString;
import com.mongodb.MongoCommandException;
import com.mongodb.MongoConfigurationException;
import com.mongodb.ReadConcern;
import com.mongodb.client.MongoClient;
Expand Down Expand Up @@ -134,17 +135,27 @@ public Stream<JsonNode> read(final String collectionName, final List<String> col
}

public Map<String, Object> getCollectionStats(final String collectionName) {
final Document collectionStats = getDatabase().runCommand(new BsonDocument("collStats", new BsonString(collectionName)));
return Map.of(COLLECTION_COUNT_KEY, collectionStats.get("count"),
COLLECTION_STORAGE_SIZE_KEY, collectionStats.get("storageSize"));
try {
final Document collectionStats = getDatabase().runCommand(new BsonDocument("collStats", new BsonString(collectionName)));
return Map.of(COLLECTION_COUNT_KEY, collectionStats.get("count"),
COLLECTION_STORAGE_SIZE_KEY, collectionStats.get("storageSize"));
} catch (final MongoCommandException e) {
LOGGER.warn("Unable to retrieve collection statistics", e);
return Map.of();
}
}

public String getServerType() {
return mongoClient.getClusterDescription().getType().name();
}

public String getServerVersion() {
return getDatabase().runCommand(new BsonDocument("buildinfo", new BsonString(""))).get("version").toString();
try {
return getDatabase().runCommand(new BsonDocument("buildinfo", new BsonString(""))).get("version").toString();
} catch (final MongoCommandException e) {
LOGGER.warn("Unable to retrieve server version", e);
return null;
}
}

private Stream<JsonNode> getStream(final MongoCursor<RawBsonDocument> cursor, final CheckedFunction<RawBsonDocument, JsonNode, Exception> mapper) {
Expand Down
33 changes: 33 additions & 0 deletions airbyte-db/db-lib/src/test/java/io/airbyte/db/mongodb/MongoDatabaseTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,17 @@
import static io.airbyte.db.mongodb.MongoDatabase.COLLECTION_STORAGE_SIZE_KEY;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import com.fasterxml.jackson.databind.JsonNode;
import com.mongodb.MongoCommandException;
import com.mongodb.ReadConcern;
import com.mongodb.ServerAddress;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoCollection;
Expand All @@ -25,6 +31,8 @@
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import org.bson.BsonDocument;
import org.bson.BsonString;
import org.bson.Document;
import org.bson.types.ObjectId;
import org.junit.jupiter.api.AfterAll;
Expand Down Expand Up @@ -155,6 +163,19 @@ void testGetCollectionStatistics() {
assertEquals(4096, statistics.get(COLLECTION_STORAGE_SIZE_KEY));
}

@Test
void testGetCollectionStatisticsCommandError() {
final MongoDatabase mongoDatabase1 = mock(MongoDatabase.class);
final com.mongodb.client.MongoDatabase clientMongoDatabase = mock(com.mongodb.client.MongoDatabase.class);
final BsonDocument response = new BsonDocument("test", new BsonString("error"));
final MongoCommandException error = new MongoCommandException(response, mock(ServerAddress.class));
when(clientMongoDatabase.runCommand(any())).thenThrow(error);
when(mongoDatabase1.getDatabase()).thenReturn(clientMongoDatabase);

final Map<String, Object> statistics = mongoDatabase1.getCollectionStats(COLLECTION_NAME);
assertTrue(statistics.isEmpty());
}

@Test
void testGetServerType() {
assertEquals(ClusterType.UNKNOWN.name(), mongoDatabase.getServerType());
Expand All @@ -165,4 +186,16 @@ void testGetServerVersion() {
assertEquals(MONGO_DB_VERSION, mongoDatabase.getServerVersion());
}

@Test
void testGetServerVersionCommandError() {
final MongoDatabase mongoDatabase1 = mock(MongoDatabase.class);
final com.mongodb.client.MongoDatabase clientMongoDatabase = mock(com.mongodb.client.MongoDatabase.class);
final BsonDocument response = new BsonDocument("test", new BsonString("error"));
final MongoCommandException error = new MongoCommandException(response, mock(ServerAddress.class));
when(clientMongoDatabase.runCommand(any())).thenThrow(error);
when(mongoDatabase1.getDatabase()).thenReturn(clientMongoDatabase);

assertNull(mongoDatabase1.getServerVersion());
}

}
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-mongodb-strict-encrypt/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,5 @@ ENV APPLICATION source-mongodb-strict-encrypt

COPY --from=build /airbyte /airbyte

LABEL io.airbyte.version=0.2.1
LABEL io.airbyte.version=0.2.2
LABEL io.airbyte.name=airbyte/source-mongodb-strict-encrypt
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-mongodb-strict-encrypt/metadata.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ data:
connectorSubtype: database
connectorType: source
definitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e
dockerImageTag: 0.2.1
dockerImageTag: 0.2.2
dockerRepository: airbyte/source-mongodb-strict-encrypt
githubIssueLabel: source-mongodb-v2
icon: mongodb.svg
Expand Down
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-mongodb-v2/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,5 @@ ENV APPLICATION source-mongodb-v2

COPY --from=build /airbyte /airbyte

LABEL io.airbyte.version=0.2.1
LABEL io.airbyte.version=0.2.2
LABEL io.airbyte.name=airbyte/source-mongodb-v2
4 changes: 2 additions & 2 deletions airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@ data:
connectorSubtype: database
connectorType: source
definitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e
dockerImageTag: 0.2.1
dockerImageTag: 0.2.2
dockerRepository: airbyte/source-mongodb-v2
githubIssueLabel: source-mongodb-v2
icon: mongodb.svg
license: ELv2
name: MongoDb
registries:
cloud:
dockerImageTag: 0.2.1
dockerImageTag: 0.2.2
dockerRepository: airbyte/source-mongodb-strict-encrypt
enabled: true
oss:
Expand Down
3 changes: 2 additions & 1 deletion docs/integrations/sources/mongodb-v2.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,8 @@ For more information regarding configuration parameters, please see [MongoDb Doc
## Changelog

| Version | Date | Pull Request | Subject |
|:--------| :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------- |
|:--------|:-----------| :------------------------------------------------------- |:----------------------------------------------------------------------------------------------------------|
| 0.2.2 | 2023-07-25 | [28692](https://github.com/airbytehq/airbyte/pull/28692) | Fix bug preventing statistics retrieval from views |
| 0.2.1 | 2023-07-21 | [28527](https://github.com/airbytehq/airbyte/pull/28527) | Log server information |
| 0.2.0 | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737) | License Update: Elv2 |
| 0.1.19 | 2022-10-07 | [17614](https://github.com/airbytehq/airbyte/pull/17614) | Increased discover performance |
Expand Down