Merge pull request #541 from QData/master

Update

docs/0_get_started/installation.md

@@ -47,3 +47,34 @@ You can also install other miscallenous optional dependencies by running
 To install both groups of packages, run
 
     pip install textattack[tensorflow,optional]
+
+
+
+## FAQ on installation
+
+For many of the dependent library issues, the following command is the first you could try: 
+```bash
+pip install --force-reinstall textattack
+```
+
+OR 
+```bash
+pip install textattack[tensorflow,optional]
+```
+
+
+Besides, we highly recommend you to use virtual environment for textattack use, 
+see [information here](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#removing-an-environment). Here is one conda example: 
+
+```bash
+conda create -n textattackenv python=3.7
+conda activate textattackenv
+conda env list
+```
+
+If you want to use the most-up-to-date version of textattack (normally with newer bug fixes), you can run the following: 
+```bash
+git clone https://github.com/QData/TextAttack.git
+cd TextAttack
+pip install .[dev]
+```

docs/apidoc/textattack.constraints.grammaticality.language_models.rst

@@ -7,6 +7,7 @@ textattack.constraints.grammaticality.language\_models package
    :show-inheritance:
 
 
+
 .. toctree::
    :maxdepth: 6
 

docs/apidoc/textattack.constraints.grammaticality.rst

@@ -7,6 +7,7 @@ textattack.constraints.grammaticality package
    :show-inheritance:
 
 
+
 .. toctree::
    :maxdepth: 6
 

docs/apidoc/textattack.constraints.rst

@@ -7,6 +7,7 @@ textattack.constraints package
    :show-inheritance:
 
 
+
 .. toctree::
    :maxdepth: 6
 

docs/apidoc/textattack.constraints.semantics.rst

@@ -7,6 +7,7 @@ textattack.constraints.semantics package
    :show-inheritance:
 
 
+
 .. toctree::
    :maxdepth: 6
 

docs/apidoc/textattack.constraints.semantics.sentence_encoders.rst

@@ -7,6 +7,7 @@ textattack.constraints.semantics.sentence\_encoders package
    :show-inheritance:
 
 
+
 .. toctree::
    :maxdepth: 6
 

docs/apidoc/textattack.datasets.rst

@@ -7,6 +7,7 @@ textattack.datasets package
    :show-inheritance:
 
 
+
 .. toctree::
    :maxdepth: 6
 

docs/apidoc/textattack.goal_functions.rst

@@ -7,6 +7,7 @@ textattack.goal\_functions package
    :show-inheritance:
 
 
+
 .. toctree::
    :maxdepth: 6
 

docs/apidoc/textattack.metrics.attack_metrics.rst

@@ -0,0 +1,26 @@
+textattack.metrics.attack\_metrics package
+==========================================
+
+.. automodule:: textattack.metrics.attack_metrics
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+
+
+.. automodule:: textattack.metrics.attack_metrics.attack_queries
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+
+.. automodule:: textattack.metrics.attack_metrics.attack_success_rate
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+
+.. automodule:: textattack.metrics.attack_metrics.words_perturbed
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/apidoc/textattack.metrics.quality_metrics.rst

@@ -0,0 +1,20 @@
+textattack.metrics.quality\_metrics package
+===========================================
+
+.. automodule:: textattack.metrics.quality_metrics
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+
+
+.. automodule:: textattack.metrics.quality_metrics.perplexity
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+
+.. automodule:: textattack.metrics.quality_metrics.use
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/apidoc/textattack.metrics.rst

@@ -0,0 +1,22 @@
+textattack.metrics package
+==========================
+
+.. automodule:: textattack.metrics
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+
+
+.. toctree::
+   :maxdepth: 6
+
+   textattack.metrics.attack_metrics
+   textattack.metrics.quality_metrics
+
+
+
+.. automodule:: textattack.metrics.metric
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/apidoc/textattack.models.rst

@@ -7,6 +7,7 @@ textattack.models package
    :show-inheritance:
 
 
+
 .. toctree::
    :maxdepth: 6
 

docs/apidoc/textattack.rst

@@ -19,6 +19,7 @@ textattack package
    textattack.goal_function_results
    textattack.goal_functions
    textattack.loggers
+   textattack.metrics
    textattack.models
    textattack.search_methods
    textattack.shared

docs/apidoc/textattack.shared.rst

@@ -7,6 +7,7 @@ textattack.shared package
    :show-inheritance:
 
 
+
 .. toctree::
    :maxdepth: 6
 

docs/apidoc/textattack.transformations.rst

@@ -7,6 +7,7 @@ textattack.transformations package
    :show-inheritance:
 
 
+
 .. toctree::
    :maxdepth: 6
 

tests/sample_outputs/run_attack_hotflip_lstm_mr_4_adv_metrics.txt

@@ -0,0 +1,74 @@
+/.*/Attack(
+  (search_method): BeamSearch(
+    (beam_width):  10
+  )
+  (goal_function):  UntargetedClassification
+  (transformation):  WordSwapGradientBased(
+    (top_n):  1
+  )
+  (constraints): 
+    (0): MaxWordsPerturbed(
+        (max_num_words):  2
+        (compare_against_original):  True
+      )
+    (1): WordEmbeddingDistance(
+        (embedding):  WordEmbedding
+        (min_cos_sim):  0.8
+        (cased):  False
+        (include_unknown_words):  True
+        (compare_against_original):  True
+      )
+    (2): PartOfSpeech(
+        (tagger_type):  nltk
+        (tagset):  universal
+        (allow_verb_noun_swap):  True
+        (compare_against_original):  True
+      )
+    (3): RepeatModification
+    (4): StopwordModification
+  (is_black_box):  False
+) 
+
+--------------------------------------------- Result 1 ---------------------------------------------
+[[Positive (96%)]] --> [[Negative (77%)]]
+
+the story gives ample opportunity for large-scale action and suspense , which director shekhar kapur [[supplies]] with tremendous skill .
+
+the story gives ample opportunity for large-scale action and suspense , which director shekhar kapur [[stagnated]] with tremendous skill .
+
+
+--------------------------------------------- Result 2 ---------------------------------------------
+[[Negative (57%)]] --> [[[SKIPPED]]]
+
+red dragon " never cuts corners .
+
+
+--------------------------------------------- Result 3 ---------------------------------------------
+[[Positive (51%)]] --> [[[FAILED]]]
+
+fresnadillo has something serious to say about the ways in which extravagant chance can distort our perspective and throw us off the path of good sense .
+
+
+--------------------------------------------- Result 4 ---------------------------------------------
+[[Positive (89%)]] --> [[[FAILED]]]
+
+throws in enough clever and unexpected twists to make the formula feel fresh .
+
+
+
++-------------------------------+--------+
+| Attack Results                |        |
++-------------------------------+--------+
+| Number of successful attacks: | 1      |
+| Number of failed attacks:     | 2      |
+| Number of skipped attacks:    | 1      |
+| Original accuracy:            | 75.0%  |
+| Accuracy under attack:        | 50.0%  |
+| Attack success rate:          | 33.33% |
+| Average perturbed word %:     | 5.56%  |
+| Average num. words per input: | 15.5   |
+| Avg num queries:              | 1.33   |
+| Average Original Perplexity:  | 291.47 |
+| Average Attack Perplexity:    | 320.33 |
+| Average Attack USE Score:     | 0.91   |
++-------------------------------+--------+

tests/sample_outputs/run_attack_transformers_datasets_adv_metrics.txt

@@ -0,0 +1,68 @@
+/.*/Attack(
+  (search_method): GreedyWordSwapWIR(
+    (wir_method):  unk
+  )
+  (goal_function):  UntargetedClassification
+  (transformation):  CompositeTransformation(
+    (0): WordSwapNeighboringCharacterSwap(
+        (random_one):  True
+      )
+    (1): WordSwapRandomCharacterSubstitution(
+        (random_one):  True
+      )
+    (2): WordSwapRandomCharacterDeletion(
+        (random_one):  True
+      )
+    (3): WordSwapRandomCharacterInsertion(
+        (random_one):  True
+      )
+    )
+  (constraints): 
+    (0): LevenshteinEditDistance(
+        (max_edit_distance):  30
+        (compare_against_original):  True
+      )
+    (1): RepeatModification
+    (2): StopwordModification
+  (is_black_box):  True
+) 
+
+--------------------------------------------- Result 1 ---------------------------------------------
+[[Negative (100%)]] --> [[Positive (71%)]]
+
+[[hide]] [[new]] secretions from the parental units 
+
+[[Ehide]] [[enw]] secretions from the parental units 
+
+
+--------------------------------------------- Result 2 ---------------------------------------------
+[[Negative (100%)]] --> [[[FAILED]]]
+
+contains no wit , only labored gags 
+
+
+--------------------------------------------- Result 3 ---------------------------------------------
+[[Positive (100%)]] --> [[Negative (96%)]]
+
+that [[loves]] its characters and communicates [[something]] [[rather]] [[beautiful]] about human nature 
+
+that [[lodes]] its characters and communicates [[somethNng]] [[rathrer]] [[beautifdul]] about human nature 
+
+
+
++-------------------------------+---------+
+| Attack Results                |         |
++-------------------------------+---------+
+| Number of successful attacks: | 2       |
+| Number of failed attacks:     | 1       |
+| Number of skipped attacks:    | 0       |
+| Original accuracy:            | 100.0%  |
+| Accuracy under attack:        | 33.33%  |
+| Attack success rate:          | 66.67%  |
+| Average perturbed word %:     | 30.95%  |
+| Average num. words per input: | 8.33    |
+| Avg num queries:              | 22.67   |
+| Average Original Perplexity:  | 1126.57 |
+| Average Attack Perplexity:    | 2823/.*/|
+| Average Attack USE Score:     | 0.76    |
++-------------------------------+---------+

tests/test_command_line/test_attack.py

@@ -48,6 +48,20 @@
         "tests/sample_outputs/run_attack_transformers_datasets.txt",
     ),
     #
+    # test loading an attack from the transformers model hub and calculate perplexity and use
+    #
+    (
+        "attack_from_transformers_adv_metrics",
+        (
+            "textattack attack --model-from-huggingface "
+            "distilbert-base-uncased-finetuned-sst-2-english "
+            "--dataset-from-huggingface glue^sst2^train --recipe deepwordbug --num-examples 3 "
+            "--enable-advance-metrics"
+            ""
+        ),
+        "tests/sample_outputs/run_attack_transformers_datasets_adv_metrics.txt",
+    ),
+    #
     # test running an attack by loading a model and dataset from file
     #
     (
@@ -72,6 +86,17 @@
         "tests/sample_outputs/run_attack_hotflip_lstm_mr_4.txt",
     ),
     #
+    # test hotflip on 10 samples from LSTM MR and calculate perplexity and use
+    #
+    (
+        "run_attack_hotflip_lstm_mr_4_adv_metrics",
+        (
+            "textattack attack --model lstm-mr --recipe hotflip "
+            "--num-examples 4 --num-examples-offset 3 --enable-advance-metrics "
+        ),
+        "tests/sample_outputs/run_attack_hotflip_lstm_mr_4_adv_metrics.txt",
+    ),
+    #
     # test: run_attack deepwordbug attack on 10 samples from LSTM MR
     #
     (