Add data attestation as a requirement

everyvoice/.schema/everyvoice-aligner-schema-0.1.json

@@ -459,6 +459,12 @@
           "title": "Label",
           "type": "string"
         },
+        "permissions_obtained": {
+          "default": false,
+          "description": "An attestation that permission has been obtained to use this data. You may not use EveryVoice to build a TTS system with data that you do not have permission to use and there are serious possible consequences for doing so. Finding data online does not constitute permission. The speaker should be aware and consent to their data being used in this way.",
+          "title": "Permissions Obtained",
+          "type": "boolean"
+        },
         "data_dir": {
           "default": "/please/create/a/path/to/your/dataset/data",
           "description": "The path to the directory with your audio files.",

everyvoice/.schema/everyvoice-shared-data-schema-0.1.json

@@ -125,6 +125,12 @@
           "title": "Label",
           "type": "string"
         },
+        "permissions_obtained": {
+          "default": false,
+          "description": "An attestation that permission has been obtained to use this data. You may not use EveryVoice to build a TTS system with data that you do not have permission to use and there are serious possible consequences for doing so. Finding data online does not constitute permission. The speaker should be aware and consent to their data being used in this way.",
+          "title": "Permissions Obtained",
+          "type": "boolean"
+        },
         "data_dir": {
           "default": "/please/create/a/path/to/your/dataset/data",
           "description": "The path to the directory with your audio files.",

everyvoice/.schema/everyvoice-spec-to-wav-schema-0.1.json

@@ -248,6 +248,12 @@
           "title": "Label",
           "type": "string"
         },
+        "permissions_obtained": {
+          "default": false,
+          "description": "An attestation that permission has been obtained to use this data. You may not use EveryVoice to build a TTS system with data that you do not have permission to use and there are serious possible consequences for doing so. Finding data online does not constitute permission. The speaker should be aware and consent to their data being used in this way.",
+          "title": "Permissions Obtained",
+          "type": "boolean"
+        },
         "data_dir": {
           "default": "/please/create/a/path/to/your/dataset/data",
           "description": "The path to the directory with your audio files.",

everyvoice/.schema/everyvoice-text-to-spec-schema-0.1.json

@@ -192,6 +192,12 @@
           "title": "Label",
           "type": "string"
         },
+        "permissions_obtained": {
+          "default": false,
+          "description": "An attestation that permission has been obtained to use this data. You may not use EveryVoice to build a TTS system with data that you do not have permission to use and there are serious possible consequences for doing so. Finding data online does not constitute permission. The speaker should be aware and consent to their data being used in this way.",
+          "title": "Permissions Obtained",
+          "type": "boolean"
+        },
         "data_dir": {
           "default": "/please/create/a/path/to/your/dataset/data",
           "description": "The path to the directory with your audio files.",

everyvoice/.schema/everyvoice-text-to-wav-schema-0.1.json

@@ -705,6 +705,12 @@
           "title": "Label",
           "type": "string"
         },
+        "permissions_obtained": {
+          "default": false,
+          "description": "An attestation that permission has been obtained to use this data. You may not use EveryVoice to build a TTS system with data that you do not have permission to use and there are serious possible consequences for doing so. Finding data online does not constitute permission. The speaker should be aware and consent to their data being used in this way.",
+          "title": "Permissions Obtained",
+          "type": "boolean"
+        },
         "data_dir": {
           "default": "/please/create/a/path/to/your/dataset/data",
           "description": "The path to the directory with your audio files.",

everyvoice/config/preprocessing_config.py

@@ -3,7 +3,7 @@
 from typing import Annotated, List, Optional, Union
 
 from annotated_types import Ge, Le
-from pydantic import Field, FilePath, ValidationInfo, model_validator
+from pydantic import Field, FilePath, ValidationInfo, field_validator, model_validator
 
 from everyvoice.config.shared_types import ConfigModel, PartialLoadConfig, init_context
 from everyvoice.config.utils import (
@@ -96,6 +96,11 @@ class AudioConfig(ConfigModel):
 
 class Dataset(PartialLoadConfig):
     label: str = Field("YourDataSet", description="A label for the source of data")
+    permissions_obtained: bool = Field(
+        False,
+        description="An attestation that permission has been obtained to use this data. You may not use EveryVoice to build a TTS system with data that you do not have permission to use and there are serious possible consequences for doing so. Finding data online does not constitute permission. The speaker should be aware and consent to their data being used in this way.",
+        validate_default=True,
+    )
     data_dir: PossiblyRelativePath = Field(
         Path("/please/create/a/path/to/your/dataset/data"),
         description="The path to the directory with your audio files.",
@@ -113,6 +118,14 @@ class Dataset(PartialLoadConfig):
         description="Advanced. A list of SoX effects to apply to your audio prior to preprocessing. Run python -c 'import torchaudio; print(torchaudio.sox_effects.effect_names())' to see a list of supported effects.",
     )
 
+    @field_validator("permissions_obtained")
+    def check_permissions(cls, permissions_obtained: bool) -> bool:
+        if not permissions_obtained:
+            raise ValueError(
+                "You are trying to run a model that does not have permission for the data it is using. Please confirm you have permission to use this data and edit your configuration file accordingly."
+            )
+        return permissions_obtained
+
 
 class PreprocessingConfig(PartialLoadConfig):
     dataset: str = Field("YourDataSet", description="The name of the dataset.")
@@ -136,7 +149,9 @@ class PreprocessingConfig(PartialLoadConfig):
         None, description="The path to an audio configuration file."
     )
     source_data: List[Dataset] = Field(
-        default_factory=lambda: [Dataset()],
+        default_factory=lambda: [
+            Dataset(permissions_obtained=True)
+        ],  # The default factory doesn't actually point to any data, so we can treat it as having permissions obtained
         description="A list of datasets.",
     )
 

everyvoice/tests/data/relative/config/everyvoice-shared-data.yaml

@@ -11,6 +11,7 @@ source_data:
   filelist: ../r-filelist.psv
   filelist_loader: everyvoice.utils.generic_psv_filelist_reader
   label: dataset_0
+  permissions_obtained: true
   sox_effects:
   - [channel, '1']
 train_split: 0.9

everyvoice/tests/preprocessed_audio_fixture.py

@@ -29,6 +29,7 @@ class PreprocessedAudioFixture:
                 Dataset(
                     data_dir=wavs_dir,
                     filelist=data_dir / "metadata.psv",
+                    permissions_obtained=True,
                 )
             ],
         ),

everyvoice/tests/test_configs.py

@@ -316,7 +316,12 @@ def test_shared_sox(self) -> None:
         vocoder_config = VocoderConfig(
             contact=self.contact,
             preprocessing=PreprocessingConfig(
-                source_data=[Dataset(), Dataset(), Dataset(), Dataset()]
+                source_data=[
+                    Dataset(permissions_obtained=True),
+                    Dataset(permissions_obtained=True),
+                    Dataset(permissions_obtained=True),
+                    Dataset(permissions_obtained=True),
+                ]
             ),
         )
         config: EveryVoiceConfig = EveryVoiceConfig(

everyvoice/tests/test_preprocessing.py

@@ -59,6 +59,14 @@ def test_run_doctest(self):
     def test_read_filelist(self):
         self.assertEqual(self.filelist[0]["basename"], "LJ050-0269")
 
+    def test_no_permissions(self):
+        no_permissions_args = self.fp_config.model_dump()
+        no_permissions_args["preprocessing"]["source_data"][0][
+            "permissions_obtained"
+        ] = False
+        with self.assertRaises(ValueError):
+            FeaturePredictionConfig(**no_permissions_args)
+
     def test_process_audio_for_alignment(self):
         config = AlignerConfig(contact=self.contact)
         for entry in self.filelist[1:]:
@@ -392,9 +400,9 @@ def test_text_processing(self):
                     preprocessed_dir.mkdir(parents=True, exist_ok=True)
                     output_filelist = preprocessed_dir / "preprocessed_filelist.psv"
                     shutil.copyfile(filelist_test_info["path"], output_filelist)
-                    fp_config.preprocessing.source_data[0].filelist = (
-                        filelist_test_info["path"]
-                    )
+                    fp_config.preprocessing.source_data[
+                        0
+                    ].filelist = filelist_test_info["path"]
                     fp_config.preprocessing.save_dir = preprocessed_dir
                     preprocessor = Preprocessor(fp_config)
                     with capture_stdout() as output, mute_logger(
@@ -489,9 +497,13 @@ def test_incremental_preprocess(self):
         with tempfile.TemporaryDirectory(
             prefix="test_incremental_preprocess", dir="."
         ) as tmpdir:
-            fp_config, lj_filelist, full_filelist, partial_filelist, to_process = (
-                self.get_simple_config(tmpdir)
-            )
+            (
+                fp_config,
+                lj_filelist,
+                full_filelist,
+                partial_filelist,
+                to_process,
+            ) = self.get_simple_config(tmpdir)
 
             fp_config.preprocessing.source_data[0].filelist = partial_filelist
             with capture_stdout() as output, mute_logger("everyvoice.preprocessor"):

everyvoice/wizard/basic.py

@@ -258,6 +258,7 @@ def effect(self):
                     filelist=new_filelist_path,
                     filelist_loader=filelist_loader,
                     sox_effects=sox_effects,
+                    permissions_obtained=True,  # If you get this far, you've answered the Dataset Permission Attestation step correctly
                 )
             )
         text_config = TextConfig(symbols=Symbols(**symbols))