Skip to content

Commit

Permalink
QA/webconnectivity: do like MK when domain is NXDOMAIN
Browse files Browse the repository at this point in the history
  • Loading branch information
bassosimone committed Aug 14, 2020
1 parent 7c5be53 commit b597576
Show file tree
Hide file tree
Showing 3 changed files with 24 additions and 8 deletions.
23 changes: 17 additions & 6 deletions QA/webconnectivity.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,6 @@ def webconnectivity_control_unreachable_http(ooni_exe, outfile):

def webconnectivity_nonexistent_domain(ooni_exe, outfile):
""" Test case where the domain does not exist """
if "measurement_kit" in ooni_exe:
return # MK result does not look correct
args = []
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
Expand All @@ -122,17 +120,30 @@ def webconnectivity_nonexistent_domain(ooni_exe, outfile):
"webconnectivity_nonexistent_domain",
args,
)
assert tk["dns_experiment_failure"] == "dns_nxdomain_error"
# TODO(bassosimone): Debateable result. We need to do better here.
# See <https://github.com/ooni/probe-engine/issues/579>.
#
# Note that MK is not doing it right here because it's suppressing the
# dns_nxdomain_error that instead is very informative. Yet, it is reporting
# a failure in HTTP, which miniooni does not because it does not make
# sense to perform HTTP when there are no IP addresses.
if "miniooni" in ooni_exe:
assert tk["dns_experiment_failure"] == "dns_nxdomain_error"
else:
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "consistent"
assert tk["control_failure"] == None
assert tk["http_experiment_failure"] == None
if "miniooni" in ooni_exe:
assert tk["http_experiment_failure"] == None
else:
assert tk["http_experiment_failure"] == "dns_lookup_error"
assert tk["body_length_match"] == None
assert tk["body_proportion"] == 0
assert tk["status_code_match"] == None
assert tk["headers_match"] == None
assert tk["title_match"] == None
assert tk["blocking"] == None
assert tk["accessible"] == None
assert tk["blocking"] == False
assert tk["accessible"] == True


def webconnectivity_tcpip_blocking_with_consistent_dns(ooni_exe, outfile):
Expand Down
5 changes: 5 additions & 0 deletions experiment/webconnectivity/summary.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,11 @@ func Summarize(tk *TestKeys) (out Summary) {
if tk.DNSExperimentFailure != nil &&
*tk.DNSExperimentFailure == modelx.FailureDNSNXDOMAINError &&
tk.DNSConsistency != nil && *tk.DNSConsistency == DNSConsistent {
// TODO(bassosimone): MK flags this as accessible. This result is debateable. We
// are doing what MK does. But we most likely want to make it better later.
//
// See <https://github.com/ooni/probe-engine/issues/579>.
out.Accessible = &accessible
return
}
// If we tried to connect more than once and never succeded and we were
Expand Down
4 changes: 2 additions & 2 deletions experiment/webconnectivity/summary_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,8 @@ func TestSummarize(t *testing.T) {
},
wantOut: webconnectivity.Summary{
BlockingReason: nil,
Blocking: nilstring,
Accessible: nil,
Blocking: false,
Accessible: &trueValue,
},
}, {
name: "with TCP total failure and consistent DNS",
Expand Down

0 comments on commit b597576

Please sign in to comment.