wip: openvpn observations

oonidata/src/oonidata/models/dataformats.py

@@ -367,23 +367,17 @@ class NetworkEvent(BaseModel):
  dial_id: Optional[int] = None
  conn_id: Optional[int] = None
 
-@add_slots
-@dataclass
-class OpenVPNConnectStatus(BaseModel):
- success: bool
- failure: Union[Failure, bool] = None
 
 @add_slots
 @dataclass
 class OpenVPNHandshake(BaseModel):
- bootstrap_time: float
+ handshake_time: float
  endpoint: str
  ip: str # we might want to make this optional, and scrub in favor of ASN/prefix
  port: int
  transport: str
  provider: str
  openvpn_options: Optional[Dict[str, str]] = None
- status: OpenVPNConnectStatus
  t0: float
  t: float
  tags: Optional[List[str]] = None

oonidata/src/oonidata/models/nettests/openvpn.py

@@ -1,7 +1,10 @@
 from dataclasses import dataclass
 from typing import List, Optional
-from oonidata.compat import add_slots
-from oonidata.models.dataformats import (
+
+from ..compat import add_slots
+
+from ..base import BaseModel
+from ..dataformats import (
  BaseTestKeys,
  Failure,
  TCPConnect,
@@ -14,12 +17,15 @@
 @add_slots
 @dataclass
 class OpenVPNTestKeys(BaseTestKeys):
- failure: Failure = None
  success: Optional[bool] = False
+ failure: Failure = None
 
  network_events: Optional[List[OpenVPNNetworkEvent]] = None
- openvpn_handshake: Optional[List[OpenVPNHandshake]] = None
  tcp_connect: Optional[List[TCPConnect]] = None
+ openvpn_handshake: Optional[List[OpenVPNHandshake]] = None
+
+ bootstrap_time: Optional[float] = None
+ tunnel: str = None
 
 
 @add_slots

oonidata/src/oonidata/models/observations.py

@@ -375,7 +375,7 @@ class HTTPMiddleboxObservation:
  table_index=("measurement_uid", "observation_id", "measurement_start_time"),
 )
 @dataclass
-class OpenVPNHandshakeObservation:
+class OpenVPNObservation:
  measurement_meta: MeasurementMeta
  probe_meta: ProbeMeta
  processing_meta: ProcessingMeta
@@ -401,8 +401,8 @@ class OpenVPNHandshakeObservation:
 
  # OpenVPN handshake observation
  openvpn_handshake_failure: Optional[Failure] = None
- openvpn_handshake_success: Optional[bool] = None
  openvpn_handshake_t: Optional[float] = None
+ openvpn_handshake_t0: Optional[float] = None
 
  # timing info about the handshake packets
  openvpn_handshake_hr_client_t: Optional[float] = None

oonipipeline/src/oonipipeline/transforms/measurement_transformer.py

@@ -25,6 +25,8 @@
  NetworkEvent,
  TCPConnect,
  TLSHandshake,
+ OpenVPNHandshake,
+ OpenVPNNetworkEvent,
  maybe_binary_data_to_bytes,
 )
 from oonidata.models.nettests.base_measurement import BaseMeasurement
@@ -36,6 +38,7 @@
  TCPObservation,
  TLSObservation,
  WebObservation,
+ OpenVPNObservation,
 )
 from oonidata.datautils import (
  InvalidCertificateChain,
@@ -760,6 +763,63 @@ def make_measurement_meta(msmt: BaseMeasurement, bucket_date: str) -> Measuremen
  measurement_start_time=measurement_start_time,
  )
 
+def count_key_exchange_packets(network_events: List[OpenVPNNetworkEvent]) -> int:
+ """
+ return number of packets exchanged in the SENT_KEY state
+ """
+ n = 0
+ for evt in network_events:
+ if evt.stage == "SENT_KEY" and evt.operation.startswith("packet_"):
+ n+=1
+ return n
+
+def measurement_to_openvpn_observation(
+ msmt_meta: MeasurementMeta,
+ probe_meta: ProbeMeta,
+ netinfodb: NetinfoDB,
+ openvpn_h: OpenVPNHandshake,
+ tcp_connect: Optional[List[TCPConnect]],
+ network_events: Optional[List[OpenVPNNetworkEvent]],
+) -> OpenVPNObservation:
+
+ oo = OpenVPNObservation(
+ measurement_meta=msmt_meta,
+ probe_meta=probe_meta,
+ failure=normalize_failure(openvpn_h.failure),
+ timestamp=make_timestamp(msmt_meta.measurement_start_time, openvpn_h.t),
+ success=openvpn_h.failure == None,
+ protocol="openvpn",
+ )
+
+ if len(tcp_connect) != 0:
+ tcp = tcp_connect[0]
+ oo.tcp_success = tcp.status.success
+ oo.tcp_failure = tcp.status.failure
+ oo.tcp_t = tcp.t
+
+ oo.handshake_failure = openvpn_h.failure
+ oo.handshake_t = openvpn_h.t
+ oo.handshake_t0 = openvpn_h.t0
+
+ for evt in network_events:
+ if evt.packet is not None:
+ if evt.packet.opcode == "P_CONTROL_HARD_RESET_CLIENT_V2":
+ oo.openvpn_handshake_hr_client_t = evt.t
+ elif evt.packet.opcode == "P_CONTROL_HARD_RESET_SERVER_V2":
+ oo.openvpn_handshake_hr_server_t = evt.t
+ elif "client_hello" in evt.tags:
+ oo.openvpn_handshake_clt_hello_t = evt.t
+ elif "server_hello" in evt.tags:
+ oo.openvpn_handshake_clt_hello_t = evt.t
+ elif evt.operation == "state" and evt.stage == "GOT_KEY":
+ oo.openvpn_handshake_got_keys__t = evt.t
+ elif evt.operation == "state" and evt.stage == "GENERATED_KEYS":
+ oo.openvpn_handshake_gen_keys__t = evt.t
+
+ oo.openvpn_handshake_key_exchg_n = count_key_exchange_packets(network_events)
+
+ return oo
+
 
 class MeasurementTransformer:
  """
@@ -911,7 +971,7 @@ def consume_web_observations(
 
  It will attempt to map them via the transaction_id or ip:port tuple.
 
- Any observation that cannot be mapped will be returned inside of it's
+ Any observation that cannot be mapped will be returned inside of its
  own WebObservation with all other columns set to None.
  """
  web_obs_list: List[WebObservation] = []
@@ -1008,5 +1068,38 @@ def consume_web_observations(
 
  return web_obs_list
 
+ def make_openvpn_observations(self,
+ tcp_connect: Optional[List[TCPConnect]],
+ openvpn_handshakes: Optional[List[OpenVPNHandshake]],
+ network_events: Optional[List[OpenVPNNetworkEvent]],
+ bootstrap_time: float,
+ ) -> List[OpenVPNObservation]:
+ """
+ Returns a list of OpenVPNObservations by mapping all related
+ TCPObservations, OpenVPNNetworkevents and OpenVPNHandshakes.
+ """
+ openvpn_obs_list: List[OpenVPNObservation] = []
+
+ for openvpn_handshake in openvpn_handshakes:
+ web_obs_list.append(
+ measurement_to_openvpn_observation(
+ msmt_meta=self.measurement_meta,
+ probe_meta=self.probe_meta,
+ netinfodb=self.netinfodb,
+ tcp_connect=tcp_connect,
+ openvpn_handshake=openvpn_handshake,
+ network_events=network_events,
+ )
+ )
+
+ # TODO: can factor out function with web_observation
+ for idx, obs in enumerate(openvpn_obs_list):
+ obs.observation_id = f"{obs.measurement_meta.measurement_uid}_{idx}"
+ obs.created_at = datetime.now(timezone.utc).replace(
+ microsecond=0, tzinfo=None
+ )
+
+ return openvpn_obs_list
+
  def make_observations(self, measurement):
  assert RuntimeError("make_observations is not implemented")

oonipipeline/src/oonipipeline/transforms/nettests/openvpn.py

@@ -0,0 +1,21 @@
+from typing import List, Tuple
+from oonidata.models.nettests import OpenVPN
+from oonidata.models.observations import OpenVPNObservation
+
+from ..measurement_transformer import MeasurementTransformer
+
+
+class OpenVPNTransformer(MeasurementTransformer):
+ def make_observations(self, msmt: OpenVPN) -> Tuple[List[OpenVPNObservation]]:
+ openvpn_obs_list = []
+ if not msmt.test_keys:
+ return (openvpn_obs_list,)
+
+ tcp_observations = self.make_tcp_observations(msmt.tcp_connect)
+
+ return self.make_openvpn_observations(
+ tcp_observations=tcp_observations,
+ openvpn_handshakes=msmt.openvpn_handshake,
+ network_events=msmt.network_events,
+ bootstrap_time=msmt.bootstrap_time,
+ )

oonipipeline/src/oonipipeline/transforms/observations.py

@@ -18,6 +18,7 @@
 from .nettests.browser_web import BrowserWebTransformer
 from .nettests.urlgetter import UrlGetterTransformer
 from .nettests.web_connectivity import WebConnectivityTransformer
+from .nettests.openvpn import OpenVPNTransformer 
 from .nettests.http_invalid_request_line import (
  HTTPInvalidRequestLineTransformer,
 )
@@ -37,6 +38,7 @@
  "http_header_field_manipulation": HTTPHeaderFieldManipulationTransformer,
  "http_invalid_request_line": HTTPInvalidRequestLineTransformer,
  "web_connectivity": WebConnectivityTransformer,
+ "openvpn": OpenVPNTransformer,
 }
 
 TypeWebConnectivityObservations = Tuple[