Update ignorable for new licenses

Reference: #139
Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

src/licensedcode/data/licenses/cc-by-sa-2.1-jp.yml

@@ -1,9 +1,11 @@
 key: cc-by-sa-2.1-jp
-owner: Creative Commons
-category: Copyleft Limited
 language: jp
 short_name: Creative Commons Attribution Share Alike 2.1 Japan
 name: Creative Commons Attribution Share Alike 2.1 Japan
+category: Copyleft Limited
+owner: Creative Commons
 spdx_license_key: CC-BY-SA-2.1-JP
 other_urls:
     - https://creativecommons.org/licenses/by-sa/2.1/jp/legalcode
+ignorable_urls:
+    - https://creativecommons.org/http:/www.creativecommons.jp/

src/licensedcode/data/licenses/cc-gpl-2.0-pt.yml

@@ -5,6 +5,10 @@ name: Creative Commons Licença Pública Geral do GNU (GPL) [General Public Lice
 category: Copyleft Limited
 owner: Creative Commons
 homepage_url: https://creativecommons.org/licenses/GPL/2.0/
+spdx_license_key: LicenseRef-scancode-cc-gpl-2.0-pt
 text_urls:
     - https://creativecommons.org/licenses/GPL/2.0/legalcode.pt
-spdx_license_key: LicenseRef-scancode-cc-gpl-2.0-pt
+ignorable_copyrights:
+    - (c) 1989, 1991 Free Software Foundation, Inc.
+ignorable_holders:
+    - Free Software Foundation, Inc.

src/licensedcode/data/licenses/cc-lgpl-2.1-pt.yml

@@ -5,7 +5,10 @@ name: Creative Commons Licença Pública Geral Menor do GNU
 category: Copyleft Limited
 owner: Creative Commons
 homepage_url: https://creativecommons.org/licenses/LGPL/2.1/
+spdx_license_key: LicenseRef-scancode-cc-lgpl-2.1-pt
 text_urls:
     - https://creativecommons.org/licenses/LGPL/2.1/legalcode.pt
-spdx_license_key: LicenseRef-scancode-cc-lgpl-2.1-pt
-
+ignorable_copyrights:
+    - Copyright (c) 1991, 1999 Free Software Foundation, Inc.
+ignorable_holders:
+    - Free Software Foundation, Inc.

src/licensedcode/data/licenses/cecill-2.1-fr.yml

@@ -5,6 +5,7 @@ category: Copyleft Limited
 owner: CeCILL
 homepage_url: http://www.cecill.info/licences/Licence_CeCILL_V2.1-fr.html
 spdx_license_key: LicenseRef-scancode-cecill-2.1-fr
+osi_license_key: CECILL-2.1
 text_urls:
     - http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.html
     - http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.txt
@@ -14,5 +15,4 @@ osi_url: http://opensource.org/licenses/CECILL-2.1
 other_urls:
     - http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.html
 ignorable_urls:
-    - http://www.cecill.info/index.en.html
-osi_license_key: CECILL-2.1
+    - http://www.cecill.info/index.fr.html

src/licensedcode/data/licenses/d-fsl-1.0-de.yml

@@ -24,9 +24,11 @@ other_urls:
     - https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl/german-free-software-license
 minimum_coverage: 10
 ignorable_copyrights:
-    - (c) Ministerium fur Wissenschaft und Forschung Nordrhein-Westfalen 2004
+    - (c) Ministerium fur Wissenschaft und Forschung Nordrhein-Westfalen 2004 Erstellt von Axel
+        Metzger und Till Jaeger
 ignorable_holders:
-    - Ministerium fur Wissenschaft und Forschung Nordrhein-Westfalen
+    - Ministerium fur Wissenschaft und Forschung Nordrhein-Westfalen Erstellt von Axel Metzger
+        und Till Jaeger
 ignorable_urls:
     - http://www.d-fsl.de/
     - http://www.fsf.org/licenses/gpl

src/licensedcode/data/licenses/dl-de-by-2-0-de.yml

@@ -5,8 +5,10 @@ name: Datenlizenz Deutschland - Namensnennung - Version 2.0 - Deutsch
 category: Permissive
 owner: govdata.de
 homepage_url: http://www.govdata.de/dl-de/by-2-0
-other_urls:
-    - https://www.dcat-ap.de/def/licenses/
 spdx_license_key: DL-DE-BY-2.0
 other_spdx_license_keys:
     - LicenseRef-scancode-dl-de-by-2-0-de
+other_urls:
+    - https://www.dcat-ap.de/def/licenses/
+ignorable_urls:
+    - http://www.govdata.de/dl-de/by-2-0

src/licensedcode/data/licenses/etalab-2.0-fr.yml

@@ -1,10 +1,12 @@
 key: etalab-2.0-fr
 language: fr
-owner: France
-category: Unstated License
 short_name: Etalab Open License 2.0 French
 name: Etalab Open License 2.0 French
+category: Unstated License
+owner: France
 spdx_license_key: etalab-2.0
 other_urls:
     - https://github.com/DISIC/politique-de-contribution-open-source/blob/master/LICENSE.pdf
     - https://raw.githubusercontent.com/DISIC/politique-de-contribution-open-source/master/LICENSE
+ignorable_urls:
+    - http://www.data.gouv.fr/fr/datasets/xxx

src/licensedcode/data/licenses/scilab-fr.yml

@@ -1,10 +1,20 @@
 key: scilab-fr
 language: fr
 short_name: SCILAB
-name: SCILAB License 
+name: SCILAB License
 category: Proprietary Free
 owner: INRIA/ENPC
 homepage_url: http://web.archive.org/web/20051212214843/http://www.scilab.org/legal/license.html
-text_urls: 
-    - https://directory.fsf.org/wiki/License:Scilab-old
 spdx_license_key: LicenseRef-scancode-scilab-fr
+text_urls:
+    - https://directory.fsf.org/wiki/License:Scilab-old
+ignorable_copyrights:
+    - (c) INRIA-ENPC
+    - Scilab (c) INRIA-ENPC
+    - Scilab (c) INRIA-ENPC.
+    - Scilab inside (c) INRIA-ENPC
+ignorable_holders:
+    - INRIA-ENPC
+    - Scilab INRIA-ENPC
+    - Scilab INRIA-ENPC.
+    - Scilab inside INRIA-ENPC

src/licensedcode/data/licenses/scola-fr.yml

@@ -2,7 +2,9 @@ key: scola-fr
 language: fr
 short_name: Statistics Canada Open Licence
 name: Entente de licence ouverte de Statistique Canada
-owner:  Statistique Canada
-homepage_url: https://www.statcan.gc.ca/fra/reference/licence
 category: Unstated License
+owner: Statistique Canada
+homepage_url: https://www.statcan.gc.ca/fra/reference/licence
 spdx_license_key: LicenseRef-scancode-scola-fr
+ignorable_emails:
+    - information@fip-pcim.gc.ca

src/licensedcode/data/rules/bsd-new_1103.RULE

@@ -0,0 +1,26 @@
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+
+    * Neither the name of the University  nor the name of 
+      Inc. nor the names of their contributors may be used to endorse or
+      promote products derived from this software without specific prior
+      written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

src/licensedcode/data/rules/bsd-new_1103.yml

@@ -0,0 +1,2 @@
+license_expression: bsd-new
+is_license_text: yes

src/licensedcode/data/rules/bsd-new_1104.RULE

@@ -0,0 +1,26 @@
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+
+    * Neither the name of the University of Cambridge nor the name of Google
+      Inc. nor the names of their contributors may be used to endorse or
+      promote products derived from this software without specific prior
+      written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

src/licensedcode/data/rules/bsd-new_1104.yml

@@ -0,0 +1,2 @@
+license_expression: bsd-new
+is_license_text: yes

src/licensedcode/data/rules/bsd-new_1105.RULE

@@ -0,0 +1,28 @@
+THE "BSD" LICENCE
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+
+    * Neither the name of the University of Cambridge nor the name of Google
+      Inc. nor the names of their contributors may be used to endorse or
+      promote products derived from this software without specific prior
+      written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

src/licensedcode/data/rules/bsd-new_1105.yml

@@ -0,0 +1,2 @@
+license_expression: bsd-new
+is_license_text: yes

src/licensedcode/models.py

@@ -802,30 +802,40 @@ def build_rules_from_licenses(licenses):
     Return an iterable of rules built from each license text from a ``licenses``
     iterable of License objects.
     """
-    for license_key, license_obj in licenses.items():
-        text_file = join(license_obj.src_dir, license_obj.text_file)
-        if exists(text_file):
-            minimum_coverage = license_obj.minimum_coverage or 0
-            yield Rule(
-                text_file=text_file,
-                license_expression=license_key,
-
-                # a license text is always 100% relevant
-                has_stored_relevance=True,
-                relevance=100,
-
-                has_stored_minimum_coverage=bool(minimum_coverage),
-                minimum_coverage=minimum_coverage,
-
-                is_from_license=True,
-                is_license_text=True,
-
-                ignorable_copyrights=license_obj.ignorable_copyrights,
-                ignorable_holders=license_obj.ignorable_holders,
-                ignorable_authors=license_obj.ignorable_authors,
-                ignorable_urls=license_obj.ignorable_urls,
-                ignorable_emails=license_obj.ignorable_emails,
-            )
+    for license_obj in licenses.values():
+        rule = build_rule_from_license(license_obj)
+        if rule:
+            yield rule
+
+
+def build_rule_from_license(license_obj):
+    """
+    Return a Rule built from a ``license`` License object, or None.
+    """
+    text_file = join(license_obj.src_dir, license_obj.text_file)
+    if exists(text_file):
+        minimum_coverage = license_obj.minimum_coverage or 0
+        return Rule(
+            text_file=text_file,
+            license_expression=license_obj.key,
+
+            # a license text is always 100% relevant
+            has_stored_relevance=True,
+            relevance=100,
+
+            has_stored_minimum_coverage=bool(minimum_coverage),
+            minimum_coverage=minimum_coverage,
+
+            is_from_license=True,
+            is_license_text=True,
+
+            ignorable_copyrights=license_obj.ignorable_copyrights,
+            ignorable_holders=license_obj.ignorable_holders,
+            ignorable_authors=license_obj.ignorable_authors,
+            ignorable_urls=license_obj.ignorable_urls,
+            ignorable_emails=license_obj.ignorable_emails,
+        )
+
 
 
 def get_all_spdx_keys(licenses_db):

tests/licensedcode/test_detection_validate.py

@@ -122,7 +122,7 @@ def check_ignorable_clues(licensish, regen=False, verbose=False):
     """
     Validate that all expected ignorable clues declared in a `licensish` License
     or Rule object are properly detected in that rule text file. Optionally
-    regen the ignorables and updates the License or Rule .yml data file.
+    ``regen`` the ignorables to update the License or Rule .yml data file.
     """
     result = models.get_ignorables(text_file=licensish.text_file)
 
@@ -132,8 +132,17 @@ def check_ignorable_clues(licensish, regen=False, verbose=False):
         pprint(result)
 
     if regen:
-        models.set_ignorables(licensish, result , verbose=verbose)
-        licensish.dump()
+        is_from_license = licensish.is_from_license
+        if is_from_license:
+            db = cache.get_licenses_db()
+            licish = db[licensish.license_expression]
+        else:
+            licish = licensish
+        models.set_ignorables(licish, result , verbose=verbose)
+        licish.dump()
+        if is_from_license:
+            licensish= models.build_rule_from_license(licish)
+
 
     expected = models.get_normalized_ignorables(licensish)