Add support for temporal cloud (#79)

Changes:
* Major changes to how the CLI works where instead of `mk*` commands we
have `schedule` and `backfill` commands
* Simplify logic for processing tls handshakes by transaction ID.

This fixes:
* #78
* #80

As part of this I am doing also a bit of refactoring of the temporal
related code to make this easier.

oonidata/src/oonidata/models/analysis.py

@@ -3,7 +3,7 @@
 from datetime import datetime
 from typing import List, Optional
 
-from oonidata.models.base import table_model, ProcessingMeta
+from .base import table_model, ProcessingMeta
 from oonidata.models.observations import MeasurementMeta, ProbeMeta
 
 

oonipipeline/Readme.md

@@ -93,12 +93,36 @@ To start the worker processes:
 hatch run oonipipeline startworkers
 ```
 
-Then you can trigger the workflow by passing the `--no-start-workers` flag:
+You should then schedule the observation generation, so that it will run every
+day.
 
+This can be accomplished by running:
+
+```
+hatch run oonipipeline schedule --create-tables
+```
+
+If you would like to also schedule the analysis, you should do:
 ```
-hatch run oonipipeline mkobs --probe-cc US --start-day 2024-01-01 --end-day 2024-01-20 --no-start-workers --create-tables
+hatch run oonipipeline schedule --analysis --create-tables
 ```
 
+These schedules can be further refined with the `--probe-cc` and `--test-name`
+options to limit the schedule to only certain countries or test names
+respectively.
+
+By navigating to the temporal web UI you will see in the schedules section these being created.
+
+You are then able to trigger a backfill (basically reprocessing the data), by
+running the following command:
+
+```
+hatch run oonipipeline backfill --schedule-id oonipipeline-observations-schedule-ALLCCS-ALLTNS --start-at 2024-01-01 --end-at 2024-02-01
+```
+
+Where the schedule-id should be taken from the output of the schedule command
+or from the temporal web UI.
+
 #### Superset
 
 Superset is a neat data viz platform.

oonipipeline/pyproject.toml

@@ -36,7 +36,8 @@ dependencies = [
  "jupyterlab ~= 4.0.7",
  "temporalio ~= 1.5.1",
  "temporalio[opentelemetry] ~= 1.5.1",
- "opentelemetry-exporter-otlp-proto-grpc ~= 1.18.0"
+ "opentelemetry-exporter-otlp-proto-grpc ~= 1.18.0",
+ "fasteners ~= 0.19",
 ]
 
 [tool.hatch.build.targets.sdist]
@@ -57,7 +58,7 @@ dependencies = [
  "memray",
  "viztracer",
  "pytest-docker",
- "ipdb"
+ "ipdb",
 ]
 python = "3.11"
 path = ".venv/"