Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: event of repos that cannot be cloned to be non-recoverable #897

Merged
merged 2 commits into from
Mar 2, 2022
Merged

fix: event of repos that cannot be cloned to be non-recoverable #897

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e79ca76
fix: event of repos that cannot be cloned to be non-recoverable
jachro Mar 2, 2022
ddecd05
fix: flaky EventHandlerSpec
jachro Mar 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ...og/src/test/scala/io/renku/eventlog/events/categories/zombieevents/EventHandlerSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ import org.scalatest.concurrent.{Eventually, IntegrationPatience}
import org.scalatest.matchers.should
import org.scalatest.wordspec.AnyWordSpec

import java.lang.Thread.sleep

class EventHandlerSpec
extends AnyWordSpec
with IOSpec
Expand Down Expand Up @@ -76,6 +78,8 @@ class EventHandlerSpec
.value
.unsafeRunSync() shouldBe Right(Accepted)

sleep(1000)

eventually {
logger.loggedOnly(
Info(
Expand Down
13 changes: 9 additions & 4 deletions ...sgenerator/events/categories/awaitinggeneration/triplesgeneration/renkulog/Commands.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,15 +167,16 @@ private object Commands {
private val authRecoverableErrors = Set("fatal: Authentication failed for")
private val nonAuthRecoverableErrors = Set(
"SSL_ERROR_SYSCALL",
"the remote end hung up unexpectedly",
"The requested URL returned error: 502",
"The requested URL returned error: 503",
"The requested URL returned error: 504",
"Error in the HTTP2 framing layer",
"HTTP/2 stream 3 was not closed cleanly before end of the underlying stream",
"Could not resolve host:",
"Host is unreachable"
)
private val malformedRepositoryErrors = Set(
"the remote end hung up unexpectedly",
"The requested URL returned error: 504",
"Error in the HTTP2 framing layer"
)
private lazy val relevantError: PartialFunction[Throwable, F[Either[ProcessingRecoverableError, Unit]]] = {
case ShelloutException(result) =>
def errorMessage(message: String) = s"git clone failed with: $message"
Expand All @@ -191,6 +192,10 @@ private object Commands {
.asLeft[Unit]
.leftWiden[ProcessingRecoverableError]
.pure[F]
case out if malformedRepositoryErrors exists out.contains =>
ProcessingNonRecoverableError
.MalformedRepository(errorMessage(result.toString()))
.raiseError[F, Either[ProcessingRecoverableError, Unit]]
case _ =>
new Exception(errorMessage(result.toString())).raiseError[F, Either[ProcessingRecoverableError, Unit]]
}
Expand Down
80 changes: 50 additions & 30 deletions ...esgenerator/events/categories/awaitinggeneration/triplesgeneration/renkulog/GitSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ import io.renku.config.ServiceUrl
import io.renku.generators.CommonGraphGenerators._
import io.renku.generators.Generators.Implicits._
import io.renku.generators.Generators._
import io.renku.graph.model.GraphModelGenerators.projectPaths
import io.renku.testtools.IOSpec
import io.renku.triplesgenerator.events.categories.ProcessingNonRecoverableError
import io.renku.triplesgenerator.events.categories.ProcessingRecoverableError._
import io.renku.triplesgenerator.events.categories.awaitinggeneration.triplesgeneration.renkulog.Commands.{GitImpl, RepositoryPath}
import org.scalamock.scalatest.MockFactory
Expand All @@ -47,24 +47,38 @@ class GitSpec extends AnyWordSpec with IOSpec with MockFactory with should.Match
git.clone(repositoryUrl, workDirectory).value.unsafeRunSync() shouldBe Right(())
}

val recoverableFailureMessagesToCheck = Set[NonBlank](
"return AuthRecoverableError if command fails with a message 'remote: HTTP Basic: Access denied'" in new TestCase {

val errorMessage = sentenceContaining(
Refined.unsafeApply("""|remote: HTTP Basic: Access denied
|fatal: Authentication failed for""".stripMargin)
).generateOne.value
val commandResultException = ShelloutException {
CommandResult(
exitCode = 1,
chunks = Seq(Left(new Bytes(errorMessage.getBytes())))
)
}
cloneCommand
.expects(repositoryUrl, destDirectory, workDirectory)
.throwing(commandResultException)

git.clone(repositoryUrl, workDirectory).value.unsafeRunSync() shouldBe Left(
AuthRecoverableError(s"git clone failed with: ${commandResultException.result.toString}")
)
}

val recoverableFailures = Set[NonBlank](
"SSL_ERROR_SYSCALL",
"fatal: the remote end hung up unexpectedly",
Refined.unsafeApply(
s"fatal: unable to access 'https://renkulab.io/gitlab/${projectPaths.generateOne}.git/': The requested URL returned error: 502"
),
"The requested URL returned error: 502",
"The requested URL returned error: 503",
"The requested URL returned error: 504",
"Error in the HTTP2 framing layer",
"HTTP/2 stream 3 was not closed cleanly before end of the underlying stream",
"Could not resolve host: renkulab.io",
"Failed to connect to renkulab.io port 443: Host is unreachable"
)
recoverableFailures foreach { error =>
s"return LogWorthyRecoverableError if command fails with a message containing '$error'" in new TestCase {

recoverableFailureMessagesToCheck foreach { recoverableError =>
s"return $LogWorthyRecoverableError if command fails with a message containing '$recoverableError'" in new TestCase {

val errorMessage = sentenceContaining(recoverableError).generateOne.value
val errorMessage = sentenceContaining(error).generateOne.value
val commandResultException = ShelloutException {
CommandResult(
exitCode = 1,
Expand All @@ -83,25 +97,31 @@ class GitSpec extends AnyWordSpec with IOSpec with MockFactory with should.Match
}
}

s"return $AuthRecoverableError if command fails with a message 'remote: HTTP Basic: Access denied'" in new TestCase {
val malformedRepoFailures = Set[NonBlank](
"fatal: the remote end hung up unexpectedly",
"The requested URL returned error: 504",
"Error in the HTTP2 framing layer"
)
malformedRepoFailures foreach { error =>
s"return ProcessingNonRecoverableError.MalformedRepository if command fails with a message containing '$error'" in new TestCase {

val errorMessage = sentenceContaining(
Refined.unsafeApply("""|remote: HTTP Basic: Access denied
|fatal: Authentication failed for""".stripMargin)
).generateOne.value
val commandResultException = ShelloutException {
CommandResult(
exitCode = 1,
chunks = Seq(Left(new Bytes(errorMessage.getBytes())))
)
}
cloneCommand
.expects(repositoryUrl, destDirectory, workDirectory)
.throwing(commandResultException)
val errorMessage = sentenceContaining(error).generateOne.value
val commandResultException = ShelloutException {
CommandResult(
exitCode = 1,
chunks = Seq(Left(new Bytes(errorMessage.getBytes())))
)
}
cloneCommand
.expects(repositoryUrl, destDirectory, workDirectory)
.throwing(commandResultException)

git.clone(repositoryUrl, workDirectory).value.unsafeRunSync() shouldBe Left(
AuthRecoverableError(s"git clone failed with: ${commandResultException.result.toString}")
)
val failure = intercept[ProcessingNonRecoverableError.MalformedRepository] {
git.clone(repositoryUrl, workDirectory).value.unsafeRunSync()
}

failure.getMessage shouldBe s"git clone failed with: ${commandResultException.result.toString}"
}
}

"fail if command fails with an unknown message" in new TestCase {
Expand Down