From 8a7c92f4638cab48ebca0c75dbc9419be35f7827 Mon Sep 17 00:00:00 2001
From: Quinten Steenhuis <qsteenhuis@suffolk.edu>
Date: Tue, 8 Nov 2022 15:41:36 -0500
Subject: [PATCH 1/8] Fix #243, don't cut off words for addendum in
 mid-sentence

---
 docassemble/AssemblyLine/al_document.py      | 66 +++++++++++---------
 docassemble/AssemblyLine/test_al_document.py | 24 ++++++-
 2 files changed, 57 insertions(+), 33 deletions(-)

diff --git a/docassemble/AssemblyLine/al_document.py b/docassemble/AssemblyLine/al_document.py
index 708ab236..ab8b3479 100644
--- a/docassemble/AssemblyLine/al_document.py
+++ b/docassemble/AssemblyLine/al_document.py
@@ -27,6 +27,7 @@
     alpha,
 )
 from docassemble.base.pdfa import pdf_to_pdfa
+from textwrap import wrap
 
 __all__ = [
     "ALAddendumField",
@@ -141,6 +142,7 @@ def overflow_value(
         preserve_newlines: bool = False,
         input_width: int = 80,
         overflow_message: str = "",
+        preserve_words: bool = True,
     ):
         """
         Try to return just the portion of the variable (list-like object or string)
@@ -157,6 +159,7 @@ def overflow_value(
             input_width=input_width,
             preserve_newlines=preserve_newlines,
             _original_value=original_value,
+            preserve_words=preserve_words,
         )
         if isinstance(safe_text, str):
             # Always get rid of double newlines, for consistency with safe_value.
@@ -201,7 +204,8 @@ def safe_value(
         overflow_message: str = "",
         input_width: int = 80,
         preserve_newlines: bool = False,
-        _original_value=None,
+        _original_value: Optional[str] = None,
+        preserve_words: bool = True,
     ):
         """
         Try to return just the portion of the variable
@@ -234,43 +238,43 @@ def safe_value(
         )
         max_chars = max(self.overflow_trigger - len(overflow_message), 0)
 
-        # If there are at least 2 lines, we can ignore overflow trigger.
-        # each line will be at least input_width wide
+        # If we preserve newlines, we need to account for max_lines, not just max_chars
         if preserve_newlines and max_lines > 1:
             if isinstance(value, str):
+                max_lines = self.max_lines(
+                    input_width=input_width, overflow_message_length=0
+                )
+                max_chars = self.overflow_trigger
                 # Replace all new line characters with just \n. \r\n inserts two lines in a PDF
                 value = re.sub(r"[\r\n]+|\r+|\n+", r"\n", value).rstrip()
-                line = 1
-                retval = ""
-                paras = value.split("\n")
-                para = 0
-                while line <= max_lines and para < len(paras):
-                    # add the whole paragraph if less than width of input
-                    if len(paras[para]) <= input_width:
-                        retval += paras[para] + "\n"
-                        line += 1
-                        para += 1
-                    else:
-                        # Keep taking the first input_width characters until we hit max_lines
-                        # or we finish the paragraph
-                        while line <= max_lines and len(paras[para]):
-                            retval += paras[para][:input_width]
-                            paras[para] = paras[para][input_width:]
-                            line += 1
-                        if not len(paras[para]):
-                            para += 1
-                            retval += "\n"
-                # TODO: check logic here to only add overflow message when we exceed length
-                if len(paras) > para:
-                    return (
-                        retval.rstrip() + overflow_message
-                    )  # remove trailing newline before adding overflow message
-                else:
-                    return retval
+                # textwrap.wrap does all the hard work for us here
+                return " ".join(
+                    wrap(
+                        value,
+                        width=input_width,
+                        max_lines=max_lines,
+                        replace_whitespace=False,
+                        placeholder=overflow_message,
+                    )
+                ).replace("  ", " ")
 
-        # Strip newlines from strings
+        # Strip newlines from strings because they take extra space
         if isinstance(value, str):
             if len(value) > self.overflow_trigger:
+                if preserve_words:
+                    return (
+                        next(
+                            iter(
+                                wrap(
+                                    value,
+                                    width=max_chars,
+                                    replace_whitespace=True,
+                                    drop_whitespace=True,
+                                )
+                            )
+                        )
+                        + overflow_message
+                    )
                 return (
                     re.sub(r"[\r\n]+|\r+|\n+", " ", value).rstrip()[:max_chars]
                     + overflow_message
diff --git a/docassemble/AssemblyLine/test_al_document.py b/docassemble/AssemblyLine/test_al_document.py
index 21ab5bd2..575a0245 100644
--- a/docassemble/AssemblyLine/test_al_document.py
+++ b/docassemble/AssemblyLine/test_al_document.py
@@ -1,6 +1,6 @@
 import unittest
-from docassemble.base.util import DAFile
-from .al_document import ALDocument, ALDocumentBundle
+from docassemble.base.util import DAFile, define
+from .al_document import ALDocument, ALDocumentBundle, ALAddendumField
 
 
 class test_dont_assume_pdf(unittest.TestCase):
@@ -32,6 +32,26 @@ def test_upload_pdf(self):
         self.assertEqual(len(new_list), 2)
         pass
 
+class test_aladdendum(unittest.TestCase):
+    def test_safe_value(self):
+        TEXT = """
+Charged by my father with a very delicate mission, I repaired, towards the end of May, 1788, to the château of Ionis, situated a dozen leagues distant, in the lands lying between Angers and Saumur. I was twenty-two, and already practising the profession of lawyer, for which I experienced but slight inclination, although neither the study of business nor of argument had presented serious difficulties to me. Taking my youth into consideration, I was not esteemed without talent, and the standing of my father, a lawyer renowned in the locality, assured me a brilliant patronage in the future, in return for any paltry efforts I might make to be worthy of replacing him. But I would have preferred literature, a more dreamy life, a more independent and more individual use of my faculties, a responsibility less submissive to the passions and interests of others. As my family was well off, and I an only son, greatly spoiled and petted, I might have chosen my own career, but I would have thus afflicted my father, who took pride in his ability to direct me 4in the road which he had cleared in advance, and I loved him too tenderly to permit my instinct to outweigh his wishes.
+
+It was a delightful evening in which I was finishing my ride on horseback through the woods that surrounded the ancient and magnificent castle of Ionis. I was well mounted, dressed en cavalier, with a species of elegance, and accompanied by a servant of whom I had not the slightest need, but whom my mother had conceived the innocent idea of giving me for the occasion, desiring that her son should present a proper appearance at the house of one of the most brilliant personages of our patronage.
+
+The night was illuminated by the soft fire of its largest stars. A slight mist veiled the scintillations of those myriads of satellites that gleam like brilliant eyes on clear, cold evenings. This was a true summer sky, pure enough to be luminous and transparent, still sufficiently softened not to overwhelm one by its immeasurable wealth. It was, if I may so speak, one of those soft firmaments that permit one to think of earth, to admire the vaporous lines of narrow horizons, to breathe without disdain its atmosphere of flowers and herbage—in fine, to consider oneself as something in this immensity, and to forget that one is but an atom in the infinite.
+
+In proportion as I approached the seigneurial park the wild perfumes of the forest were mingled with those of the lilacs and acacias, whose blooming heads leaned over the wall. Soon through the shrubbery I saw the windows of the manor gleaming behind their curtains of purple moire, divided by the dark crossbars of the frame work. It was a magnificent castle 5of the renaissance, a chef-d’œuvre of taste mingled with caprice, one of those dwellings where one is impressed by something indescribably ingenious and bold, which from the imagination of the architect seems to pass into one’s own, and take possession of it, raising it above the usages and preoccupations of a positive world.
+
+I confess that my heart beat fast in giving my name to the lackey commissioned to announce me. I had never seen Madame d’Ionis; she passed for one of the prettiest women in the country, was twenty-two, and had a husband who was neither handsome nor amiable, and who neglected her in order to travel. Her writing was charming, and she found means to show not only a great deal of sense, but still more cleverness in her business letters. Altogether she was a very fine character. This was all that I knew of her, and it was sufficient for me to dread appearing awkward or provincial. I grew pale on entering the salon. My first impression then was one of relief and pleasure, when I found myself in the presence of two stout and very ugly old women, one of whom, Madame the Dowager d’Ionis informed me that her daughter-in-law was at the house of her friends in the neighborhood, and probably would not return before the next day.
+
+“You are welcome, all the same,” added this matron. “We have a very friendly and grateful feeling for your father, and it appears that we stand in great need of his counsel, which you are without doubt charged to communicate to us.”
+
+“I came from him,” I replied, “to talk over the affair with Madame d’Ionis.”        
+        """
+        define("fake_value",TEXT)
+        myfield = ALAddendumField(field_name="fake_value", overflow_trigger=120)
+        self.assertLessEqual(len(myfield.safe_value(overflow_message=" [See addendum]")), 120)
 
 if __name__ == "__main__":
     unittest.main()

From 6733f21239e09163eff0bed54ff5b9cffd68fd01 Mon Sep 17 00:00:00 2001
From: nonprofittechy <nonprofittechy@users.noreply.github.com>
Date: Tue, 8 Nov 2022 20:42:42 +0000
Subject: [PATCH 2/8] :art: Format Python code with psf/black

---
 docassemble/AssemblyLine/test_al_document.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/docassemble/AssemblyLine/test_al_document.py b/docassemble/AssemblyLine/test_al_document.py
index 575a0245..b4dad2bd 100644
--- a/docassemble/AssemblyLine/test_al_document.py
+++ b/docassemble/AssemblyLine/test_al_document.py
@@ -32,6 +32,7 @@ def test_upload_pdf(self):
         self.assertEqual(len(new_list), 2)
         pass
 
+
 class test_aladdendum(unittest.TestCase):
     def test_safe_value(self):
         TEXT = """
@@ -49,9 +50,12 @@ def test_safe_value(self):
 
 “I came from him,” I replied, “to talk over the affair with Madame d’Ionis.”        
         """
-        define("fake_value",TEXT)
+        define("fake_value", TEXT)
         myfield = ALAddendumField(field_name="fake_value", overflow_trigger=120)
-        self.assertLessEqual(len(myfield.safe_value(overflow_message=" [See addendum]")), 120)
+        self.assertLessEqual(
+            len(myfield.safe_value(overflow_message=" [See addendum]")), 120
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

From 00d698a522b4fb4cda69e796755e657c38d57ddc Mon Sep 17 00:00:00 2001
From: Quinten Steenhuis <qsteenhuis@gmail.com>
Date: Tue, 8 Nov 2022 16:43:05 -0500
Subject: [PATCH 3/8] Multi-line return value

---
 docassemble/AssemblyLine/al_document.py | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/docassemble/AssemblyLine/al_document.py b/docassemble/AssemblyLine/al_document.py
index ab8b3479..ce90b7f0 100644
--- a/docassemble/AssemblyLine/al_document.py
+++ b/docassemble/AssemblyLine/al_document.py
@@ -262,19 +262,13 @@ def safe_value(
         if isinstance(value, str):
             if len(value) > self.overflow_trigger:
                 if preserve_words:
-                    return (
-                        next(
-                            iter(
-                                wrap(
-                                    value,
-                                    width=max_chars,
-                                    replace_whitespace=True,
-                                    drop_whitespace=True,
-                                )
-                            )
-                        )
-                        + overflow_message
+                    retval = wrap(
+                        value,
+                        width=max_chars,
+                        replace_whitespace=True,
+                        drop_whitespace=True,
                     )
+                    return next(iter(retval)) + overflow_message
                 return (
                     re.sub(r"[\r\n]+|\r+|\n+", " ", value).rstrip()[:max_chars]
                     + overflow_message

From 6a72c823979a28f84d47c07ec23a60e57d6c053f Mon Sep 17 00:00:00 2001
From: Quinten Steenhuis <qsteenhuis@gmail.com>
Date: Thu, 10 Nov 2022 11:54:30 -0500
Subject: [PATCH 4/8] Improve calculation of max lines, better test cases,
 consistent overflow and safe_value handling of whitespace

---
 docassemble/AssemblyLine/al_document.py      | 67 +++++++++++++-------
 docassemble/AssemblyLine/test_al_document.py | 35 ++++++++--
 2 files changed, 72 insertions(+), 30 deletions(-)

diff --git a/docassemble/AssemblyLine/al_document.py b/docassemble/AssemblyLine/al_document.py
index ce90b7f0..329252c2 100644
--- a/docassemble/AssemblyLine/al_document.py
+++ b/docassemble/AssemblyLine/al_document.py
@@ -28,6 +28,7 @@
 )
 from docassemble.base.pdfa import pdf_to_pdfa
 from textwrap import wrap
+from math import floor
 
 __all__ = [
     "ALAddendumField",
@@ -147,6 +148,13 @@ def overflow_value(
         """
         Try to return just the portion of the variable (list-like object or string)
         that is not contained in the safe_value().
+
+        Whitespace will be altered. If preserve_newlines is true, the return value may have newlines,
+        but double newlines and Windows style (\r\n) will be replaced with \n. Double spaces will replaced
+        with a single space.
+
+        If preserve_newlines is false, all whitespace, including newlines and tabs, will be replaced
+        with a single space.
         """
         # Handle a Boolean overflow first
         if isinstance(self.overflow_trigger, bool) and self.overflow_trigger:
@@ -161,11 +169,18 @@ def overflow_value(
             _original_value=original_value,
             preserve_words=preserve_words,
         )
-        if isinstance(safe_text, str):
-            # Always get rid of double newlines, for consistency with safe_value.
-            value_to_process = re.sub(
-                r"[\r\n]+|\r+|\n+", r"\n", original_value
-            ).rstrip()
+        if isinstance(safe_text, str):               
+            max_lines = self.max_lines(input_width=input_width)
+
+            if preserve_newlines and max_lines > 1:
+                # we do our own substitution of whitespace, only double newlines and spaces
+                value_to_process = re.sub(
+                    r"[\r\n]+|\r+|\n+", r"\n", original_value
+                ).rstrip().replace("  ", " ")
+            else:
+                # textwrap.wrap(replace_whitespace=True) replaces all whitespace, not just double newlines and spaces
+                value_to_process = re.sub(r"\s+", " ", original_value).rstrip()
+
             if safe_text == value_to_process:  # no overflow
                 return ""
             # If this is a string, the safe value will include an overflow message. Delete
@@ -182,14 +197,11 @@ def overflow_value(
         # Do not subtract length of overflow message if this is a list of objects instead of a string
         return original_value[self.overflow_trigger :]
 
-    def max_lines(self, input_width: int = 80, overflow_message_length=0) -> int:
+    def max_lines(self, input_width: int = 80) -> int:
         """
-        Estimate the number of rows in the field in the output document.
+        Calculate the number of lines of text that will fit in the specified input
         """
-        return (
-            int(max(self.overflow_trigger - overflow_message_length, 0) / input_width)
-            + 1
-        )
+        return floor(self.overflow_trigger / input_width)
 
     def value(self) -> Any:
         """
@@ -206,10 +218,22 @@ def safe_value(
         preserve_newlines: bool = False,
         _original_value: Optional[str] = None,
         preserve_words: bool = True,
-    ):
+    ) -> Union[str, List[Any]]:
         """
-        Try to return just the portion of the variable
-        that is _shorter than_ the overflow trigger. Otherwise, return empty string.
+        Return just the portion of the variable that heuristics suggest will fit in the specified overflow_trigger
+        limit. If the value is not defined, return empty string.
+
+        When `preserve_newlines` is `True`, the output will be limited to a number of lines, not a number
+        of characters. The max number of lines will be calculated as `floor(self.overflow_trigger/input_width)`.
+        Therefore, it is important that `input_width` is a divisor of `overflow_trigger`.
+
+        Whitespace will be altered. If preserve_newlines is true, the return value may have newlines,
+        but double newlines and Windows style (\r\n) will be replaced with \n. Double spaces will replaced
+        with a single space.
+
+        If preserve_newlines is false, all whitespace, including newlines and tabs, will be replaced
+        with a single space.
+
         Args:
             overflow_message (str): A short message to go on the page where text is cutoff.
             input_width (int): The width, in characters, of the input box. Defaults to 80.
@@ -234,17 +258,14 @@ def safe_value(
             return value
 
         max_lines = self.max_lines(
-            input_width=input_width, overflow_message_length=len(overflow_message)
+            input_width=input_width
         )
         max_chars = max(self.overflow_trigger - len(overflow_message), 0)
 
-        # If we preserve newlines, we need to account for max_lines, not just max_chars
-        if preserve_newlines and max_lines > 1:
-            if isinstance(value, str):
-                max_lines = self.max_lines(
-                    input_width=input_width, overflow_message_length=0
-                )
-                max_chars = self.overflow_trigger
+        # Strip newlines from strings because they take extra space
+        if isinstance(value, str):
+            # If we preserve newlines, we need to account for max_lines, not just max_chars
+            if preserve_newlines and max_lines > 1:
                 # Replace all new line characters with just \n. \r\n inserts two lines in a PDF
                 value = re.sub(r"[\r\n]+|\r+|\n+", r"\n", value).rstrip()
                 # textwrap.wrap does all the hard work for us here
@@ -258,8 +279,6 @@ def safe_value(
                     )
                 ).replace("  ", " ")
 
-        # Strip newlines from strings because they take extra space
-        if isinstance(value, str):
             if len(value) > self.overflow_trigger:
                 if preserve_words:
                     retval = wrap(
diff --git a/docassemble/AssemblyLine/test_al_document.py b/docassemble/AssemblyLine/test_al_document.py
index b4dad2bd..bf27773c 100644
--- a/docassemble/AssemblyLine/test_al_document.py
+++ b/docassemble/AssemblyLine/test_al_document.py
@@ -35,8 +35,7 @@ def test_upload_pdf(self):
 
 class test_aladdendum(unittest.TestCase):
     def test_safe_value(self):
-        TEXT = """
-Charged by my father with a very delicate mission, I repaired, towards the end of May, 1788, to the château of Ionis, situated a dozen leagues distant, in the lands lying between Angers and Saumur. I was twenty-two, and already practising the profession of lawyer, for which I experienced but slight inclination, although neither the study of business nor of argument had presented serious difficulties to me. Taking my youth into consideration, I was not esteemed without talent, and the standing of my father, a lawyer renowned in the locality, assured me a brilliant patronage in the future, in return for any paltry efforts I might make to be worthy of replacing him. But I would have preferred literature, a more dreamy life, a more independent and more individual use of my faculties, a responsibility less submissive to the passions and interests of others. As my family was well off, and I an only son, greatly spoiled and petted, I might have chosen my own career, but I would have thus afflicted my father, who took pride in his ability to direct me 4in the road which he had cleared in advance, and I loved him too tenderly to permit my instinct to outweigh his wishes.
+        text_testcase1 = """Charged by my father with a very delicate mission, I repaired, towards the end of May, 1788, to the château of Ionis, situated a dozen leagues distant, in the lands lying between Angers and Saumur. I was twenty-two, and already practising the profession of lawyer, for which I experienced but slight inclination, although neither the study of business nor of argument had presented serious difficulties to me. Taking my youth into consideration, I was not esteemed without talent, and the standing of my father, a lawyer renowned in the locality, assured me a brilliant patronage in the future, in return for any paltry efforts I might make to be worthy of replacing him. But I would have preferred literature, a more dreamy life, a more independent and more individual use of my faculties, a responsibility less submissive to the passions and interests of others. As my family was well off, and I an only son, greatly spoiled and petted, I might have chosen my own career, but I would have thus afflicted my father, who took pride in his ability to direct me 4in the road which he had cleared in advance, and I loved him too tenderly to permit my instinct to outweigh his wishes.
 
 It was a delightful evening in which I was finishing my ride on horseback through the woods that surrounded the ancient and magnificent castle of Ionis. I was well mounted, dressed en cavalier, with a species of elegance, and accompanied by a servant of whom I had not the slightest need, but whom my mother had conceived the innocent idea of giving me for the occasion, desiring that her son should present a proper appearance at the house of one of the most brilliant personages of our patronage.
 
@@ -48,12 +47,36 @@ def test_safe_value(self):
 
 “You are welcome, all the same,” added this matron. “We have a very friendly and grateful feeling for your father, and it appears that we stand in great need of his counsel, which you are without doubt charged to communicate to us.”
 
-“I came from him,” I replied, “to talk over the affair with Madame d’Ionis.”        
+“I came from him,” I replied, “to talk over the affair with Madame d’Ionis.”
         """
-        define("fake_value", TEXT)
-        myfield = ALAddendumField(field_name="fake_value", overflow_trigger=120)
+        define("testcase1", text_testcase1)
+        myfield = ALAddendumField(field_name="testcase1")
+        myfield.overflow_trigger = 120
         self.assertLessEqual(
-            len(myfield.safe_value(overflow_message=" [See addendum]")), 120
+            len(myfield.safe_value(input_width=60, overflow_message=" [See addendum]")), 120
+        )
+
+        myfield.overflow_trigger = 160
+        self.assertTrue(
+            myfield.safe_value(overflow_message="").endswith("in the")
+        )
+
+        self.assertTrue(myfield.overflow_value(overflow_message="").startswith("lands"))
+
+        self.assertEqual(
+            len(myfield.safe_value(overflow_message="")), 158
+        )
+
+        text_with_weird_spaces = """Testing here
+
+with some very short words, but a whole lot of them, so it'll be over the overflow, over the flow yah know?"""
+        define("with_weird_spaces", text_with_weird_spaces)
+        field_with_weird_spaces = ALAddendumField(field_name="with_weird_spaces", overflow_trigger=23)
+        self.assertTrue(
+            field_with_weird_spaces.safe_value(overflow_message="").endswith("with")
+        )
+        self.assertTrue(
+            field_with_weird_spaces.overflow_value(overflow_message="").startswith("some")
         )
 
 

From 8506830dd2024b178862f8614c6d8dbacb8b02f8 Mon Sep 17 00:00:00 2001
From: nonprofittechy <nonprofittechy@users.noreply.github.com>
Date: Thu, 10 Nov 2022 16:54:57 +0000
Subject: [PATCH 5/8] :art: Format Python code with psf/black

---
 docassemble/AssemblyLine/al_document.py      | 14 +++++++-------
 docassemble/AssemblyLine/test_al_document.py | 19 ++++++++++---------
 2 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/docassemble/AssemblyLine/al_document.py b/docassemble/AssemblyLine/al_document.py
index 329252c2..a1c2827d 100644
--- a/docassemble/AssemblyLine/al_document.py
+++ b/docassemble/AssemblyLine/al_document.py
@@ -169,14 +169,16 @@ def overflow_value(
             _original_value=original_value,
             preserve_words=preserve_words,
         )
-        if isinstance(safe_text, str):               
+        if isinstance(safe_text, str):
             max_lines = self.max_lines(input_width=input_width)
 
             if preserve_newlines and max_lines > 1:
                 # we do our own substitution of whitespace, only double newlines and spaces
-                value_to_process = re.sub(
-                    r"[\r\n]+|\r+|\n+", r"\n", original_value
-                ).rstrip().replace("  ", " ")
+                value_to_process = (
+                    re.sub(r"[\r\n]+|\r+|\n+", r"\n", original_value)
+                    .rstrip()
+                    .replace("  ", " ")
+                )
             else:
                 # textwrap.wrap(replace_whitespace=True) replaces all whitespace, not just double newlines and spaces
                 value_to_process = re.sub(r"\s+", " ", original_value).rstrip()
@@ -257,9 +259,7 @@ def safe_value(
         ):
             return value
 
-        max_lines = self.max_lines(
-            input_width=input_width
-        )
+        max_lines = self.max_lines(input_width=input_width)
         max_chars = max(self.overflow_trigger - len(overflow_message), 0)
 
         # Strip newlines from strings because they take extra space
diff --git a/docassemble/AssemblyLine/test_al_document.py b/docassemble/AssemblyLine/test_al_document.py
index bf27773c..7fe91c43 100644
--- a/docassemble/AssemblyLine/test_al_document.py
+++ b/docassemble/AssemblyLine/test_al_document.py
@@ -53,30 +53,31 @@ def test_safe_value(self):
         myfield = ALAddendumField(field_name="testcase1")
         myfield.overflow_trigger = 120
         self.assertLessEqual(
-            len(myfield.safe_value(input_width=60, overflow_message=" [See addendum]")), 120
+            len(myfield.safe_value(input_width=60, overflow_message=" [See addendum]")),
+            120,
         )
 
         myfield.overflow_trigger = 160
-        self.assertTrue(
-            myfield.safe_value(overflow_message="").endswith("in the")
-        )
+        self.assertTrue(myfield.safe_value(overflow_message="").endswith("in the"))
 
         self.assertTrue(myfield.overflow_value(overflow_message="").startswith("lands"))
 
-        self.assertEqual(
-            len(myfield.safe_value(overflow_message="")), 158
-        )
+        self.assertEqual(len(myfield.safe_value(overflow_message="")), 158)
 
         text_with_weird_spaces = """Testing here
 
 with some very short words, but a whole lot of them, so it'll be over the overflow, over the flow yah know?"""
         define("with_weird_spaces", text_with_weird_spaces)
-        field_with_weird_spaces = ALAddendumField(field_name="with_weird_spaces", overflow_trigger=23)
+        field_with_weird_spaces = ALAddendumField(
+            field_name="with_weird_spaces", overflow_trigger=23
+        )
         self.assertTrue(
             field_with_weird_spaces.safe_value(overflow_message="").endswith("with")
         )
         self.assertTrue(
-            field_with_weird_spaces.overflow_value(overflow_message="").startswith("some")
+            field_with_weird_spaces.overflow_value(overflow_message="").startswith(
+                "some"
+            )
         )
 
 

From e2dc43d6d3412940d2a0c028f6e128edc4a1f4f5 Mon Sep 17 00:00:00 2001
From: Quinten Steenhuis <qsteenhuis@gmail.com>
Date: Thu, 10 Nov 2022 12:32:41 -0500
Subject: [PATCH 6/8] Test case

---
 docassemble/AssemblyLine/al_document.py      | 22 ++++++++++++--------
 docassemble/AssemblyLine/test_al_document.py |  2 --
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/docassemble/AssemblyLine/al_document.py b/docassemble/AssemblyLine/al_document.py
index a1c2827d..39f5f582 100644
--- a/docassemble/AssemblyLine/al_document.py
+++ b/docassemble/AssemblyLine/al_document.py
@@ -269,15 +269,19 @@ def safe_value(
                 # Replace all new line characters with just \n. \r\n inserts two lines in a PDF
                 value = re.sub(r"[\r\n]+|\r+|\n+", r"\n", value).rstrip()
                 # textwrap.wrap does all the hard work for us here
-                return " ".join(
-                    wrap(
-                        value,
-                        width=input_width,
-                        max_lines=max_lines,
-                        replace_whitespace=False,
-                        placeholder=overflow_message,
+                return (
+                    " ".join(
+                        wrap(
+                            value,
+                            width=input_width,
+                            max_lines=max_lines,
+                            replace_whitespace=False,
+                            placeholder=overflow_message,
+                        )
                     )
-                ).replace("  ", " ")
+                    .replace("  ", " ")
+                    .rstrip()
+                )
 
             if len(value) > self.overflow_trigger:
                 if preserve_words:
@@ -287,7 +291,7 @@ def safe_value(
                         replace_whitespace=True,
                         drop_whitespace=True,
                     )
-                    return next(iter(retval)) + overflow_message
+                    return next(iter(retval)).rstrip() + overflow_message
                 return (
                     re.sub(r"[\r\n]+|\r+|\n+", " ", value).rstrip()[:max_chars]
                     + overflow_message
diff --git a/docassemble/AssemblyLine/test_al_document.py b/docassemble/AssemblyLine/test_al_document.py
index 7fe91c43..456c5ff9 100644
--- a/docassemble/AssemblyLine/test_al_document.py
+++ b/docassemble/AssemblyLine/test_al_document.py
@@ -59,9 +59,7 @@ def test_safe_value(self):
 
         myfield.overflow_trigger = 160
         self.assertTrue(myfield.safe_value(overflow_message="").endswith("in the"))
-
         self.assertTrue(myfield.overflow_value(overflow_message="").startswith("lands"))
-
         self.assertEqual(len(myfield.safe_value(overflow_message="")), 158)
 
         text_with_weird_spaces = """Testing here

From 64618650b16b38f3634a0f516add1e6dc3dbba37 Mon Sep 17 00:00:00 2001
From: Quinten Steenhuis <qsteenhuis@gmail.com>
Date: Thu, 10 Nov 2022 15:35:38 -0500
Subject: [PATCH 7/8] Some correctness fixes for whitespace, working tests
 without 'define()'

---
 docassemble/AssemblyLine/al_document.py      | 23 +++++--------
 docassemble/AssemblyLine/test_al_document.py | 36 +++++++++-----------
 2 files changed, 26 insertions(+), 33 deletions(-)

diff --git a/docassemble/AssemblyLine/al_document.py b/docassemble/AssemblyLine/al_document.py
index 39f5f582..8a06a64c 100644
--- a/docassemble/AssemblyLine/al_document.py
+++ b/docassemble/AssemblyLine/al_document.py
@@ -25,6 +25,7 @@
     space_to_underscore,
     DAStaticFile,
     alpha,
+    showifdef,
 )
 from docassemble.base.pdfa import pdf_to_pdfa
 from textwrap import wrap
@@ -175,13 +176,11 @@ def overflow_value(
             if preserve_newlines and max_lines > 1:
                 # we do our own substitution of whitespace, only double newlines and spaces
                 value_to_process = (
-                    re.sub(r"[\r\n]+|\r+|\n+", r"\n", original_value)
-                    .rstrip()
-                    .replace("  ", " ")
-                )
+                    re.sub(r"[\r\n]+|\r+|\n+", r"\n", original_value).replace("  ", " ")
+                ).rstrip()
             else:
                 # textwrap.wrap(replace_whitespace=True) replaces all whitespace, not just double newlines and spaces
-                value_to_process = re.sub(r"\s+", " ", original_value).rstrip()
+                value_to_process = re.sub(r"\s+", " ", original_value).strip()
 
             if safe_text == value_to_process:  # no overflow
                 return ""
@@ -194,7 +193,7 @@ def overflow_value(
             #   1. We replace all double newlines with \n.
             #   2. Character count will adjust to reflect double-newlines being replaced with one char.
             overflow_start = max(len(safe_text) - len(overflow_message), 0)
-            return value_to_process[overflow_start:]
+            return value_to_process[overflow_start:].lstrip()
 
         # Do not subtract length of overflow message if this is a list of objects instead of a string
         return original_value[self.overflow_trigger :]
@@ -283,6 +282,7 @@ def safe_value(
                     .rstrip()
                 )
 
+            value = re.sub(r"\s+", " ", value)
             if len(value) > self.overflow_trigger:
                 if preserve_words:
                     retval = wrap(
@@ -292,12 +292,9 @@ def safe_value(
                         drop_whitespace=True,
                     )
                     return next(iter(retval)).rstrip() + overflow_message
-                return (
-                    re.sub(r"[\r\n]+|\r+|\n+", " ", value).rstrip()[:max_chars]
-                    + overflow_message
-                )
+                return value.rstrip()[:max_chars] + overflow_message
             else:
-                return re.sub(r"[\r\n]+|\r+|\n+", " ", value).rstrip()[:max_chars]
+                return re.sub(r"\s+", " ", value).rstrip()
 
         # If the overflow item is a list or DAList
         if isinstance(value, list) or isinstance(value, DAList):
@@ -311,9 +308,7 @@ def value_if_defined(self) -> Any:
         Return the value of the field if it is defined, otherwise return an empty string.
         Addendum should never trigger docassemble's variable gathering.
         """
-        if defined(self.field_name):
-            return value(self.field_name)
-        return ""
+        return showifdef(self.field_name, "")
 
     def __str__(self):
         return str(self.value_if_defined())
diff --git a/docassemble/AssemblyLine/test_al_document.py b/docassemble/AssemblyLine/test_al_document.py
index 456c5ff9..0c07439c 100644
--- a/docassemble/AssemblyLine/test_al_document.py
+++ b/docassemble/AssemblyLine/test_al_document.py
@@ -1,5 +1,5 @@
 import unittest
-from docassemble.base.util import DAFile, define
+from docassemble.base.util import DAFile
 from .al_document import ALDocument, ALDocumentBundle, ALAddendumField
 
 
@@ -49,34 +49,32 @@ def test_safe_value(self):
 
 “I came from him,” I replied, “to talk over the affair with Madame d’Ionis.”
         """
-        define("testcase1", text_testcase1)
+        testcase1 = text_testcase1
         myfield = ALAddendumField(field_name="testcase1")
-        myfield.overflow_trigger = 120
+        myfield.overflow_trigger = 160
         self.assertLessEqual(
-            len(myfield.safe_value(input_width=60, overflow_message=" [See addendum]")),
-            120,
+            len(myfield.safe_value(overflow_message=" [See addendum]")),
+            160,
         )
-
-        myfield.overflow_trigger = 160
-        self.assertTrue(myfield.safe_value(overflow_message="").endswith("in the"))
-        self.assertTrue(myfield.overflow_value(overflow_message="").startswith("lands"))
+        # print(myfield.safe_value(overflow_message=""))
+        safe_value = myfield.safe_value(overflow_message="")
+        self.assertTrue(safe_value.endswith("in the"))
+        overflow_value = myfield.overflow_value(overflow_message="")
+        self.assertTrue(overflow_value.startswith("lands"))
         self.assertEqual(len(myfield.safe_value(overflow_message="")), 158)
 
-        text_with_weird_spaces = """Testing here
+        with_weird_spaces = """Testing here
+
 
 with some very short words, but a whole lot of them, so it'll be over the overflow, over the flow yah know?"""
-        define("with_weird_spaces", text_with_weird_spaces)
+
         field_with_weird_spaces = ALAddendumField(
             field_name="with_weird_spaces", overflow_trigger=23
         )
-        self.assertTrue(
-            field_with_weird_spaces.safe_value(overflow_message="").endswith("with")
-        )
-        self.assertTrue(
-            field_with_weird_spaces.overflow_value(overflow_message="").startswith(
-                "some"
-            )
-        )
+        safe_value = field_with_weird_spaces.safe_value(overflow_message="")
+        self.assertTrue(safe_value.endswith("some"), msg=f"{ safe_value }")
+        overflow_value = field_with_weird_spaces.overflow_value(overflow_message="")
+        self.assertTrue(overflow_value.startswith("very"), msg=f"{ overflow_value }")
 
 
 if __name__ == "__main__":

From 7d6a8d458b7c404c6e9698ed3f06aca552670ca2 Mon Sep 17 00:00:00 2001
From: Quinten Steenhuis <qsteenhuis@gmail.com>
Date: Thu, 10 Nov 2022 15:48:00 -0500
Subject: [PATCH 8/8] Update docassemble/AssemblyLine/al_document.py

---
 docassemble/AssemblyLine/al_document.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docassemble/AssemblyLine/al_document.py b/docassemble/AssemblyLine/al_document.py
index 8a06a64c..a50a2e6f 100644
--- a/docassemble/AssemblyLine/al_document.py
+++ b/docassemble/AssemblyLine/al_document.py
@@ -294,7 +294,7 @@ def safe_value(
                     return next(iter(retval)).rstrip() + overflow_message
                 return value.rstrip()[:max_chars] + overflow_message
             else:
-                return re.sub(r"\s+", " ", value).rstrip()
+                return value.rstrip()
 
         # If the overflow item is a list or DAList
         if isinstance(value, list) or isinstance(value, DAList):