Trac #29919: Restore "huge" package type

(from #29901)

... for packages like `database_stein_watkins` and `polytopes_db_4d`

Parts of `sage_bootstrap` know about this package type, but other
scripts don't.

Our GH Actions scripts filter out these packages by name. This can be
refactored.

With the present ticket, we can now say `sage --package list :optional:
:experimental: --no-file huge` to filter out the huge packages.

Huge packages are now marked by the presence of the file `huge`; so this
is actually orthogonal to the package type.

We also add files named `has_nonfree_dependencies` to some other
packages so that they can be filtered out by the same mechanism.

URL: https://trac.sagemath.org/29919
Reported by: mkoeppe
Ticket author(s): Matthias Koeppe
Reviewer(s): John Palmieri

build/sage_bootstrap/cmdline.py

@@ -201,12 +201,16 @@ def make_parser():
         'package_class', metavar='[package_name|:package_type:]',
         type=str, default=[':all:'], nargs='*',
         help=('package name or designator for all packages of a given type '
-              '(one of :all:, :standard:, :optional:, :experimental:, and :huge:); '
+              '(one of :all:, :standard:, :optional:, and :experimental:); '
               'default: :all:'))
     parser_list.add_argument(
         '--has-file', action='append', default=[], metavar='FILENAME', dest='has_files',
-        help=('only include packages that have this file in their metadata directory'
+        help=('only include packages that have this file in their metadata directory '
               '(examples: SPKG.rst, spkg-configure.m4, distros/debian.txt)'))
+    parser_list.add_argument(
+        '--no-file', action='append', default=[], metavar='FILENAME', dest='no_files',
+        help=('only include packages that do not have this file in their metadata directory '
+              '(examples: huge, patches)'))
     parser_list.add_argument(
         '--exclude', action='append', default=[], metavar='PACKAGE_NAME',
         help='exclude package from list')
@@ -282,7 +286,7 @@ def make_parser():
         'package_class', metavar='[package_name|:package_type:]',
         type=str, default=[':all:'], nargs='*',
         help=('package name or designator for all packages of a given type '
-              '(one of :all:, :standard:, :optional:, :experimental:, and :huge:); '
+              '(one of :all:, :standard:, :optional:, and :experimental:); '
               'default: :all:'))
 
     parser_create = subparsers.add_parser(
@@ -330,7 +334,7 @@ def run():
     if args.subcommand == 'config':
         app.config()
     elif args.subcommand == 'list':
-        app.list_cls(*args.package_class, has_files=args.has_files, exclude=args.exclude)
+        app.list_cls(*args.package_class, has_files=args.has_files, no_files=args.no_files, exclude=args.exclude)
     elif args.subcommand == 'name':
         app.name(args.tarball_filename)
     elif args.subcommand == 'tarball':

build/sage_bootstrap/expand_class.py

@@ -24,14 +24,18 @@ class PackageClass(object):
     def __init__(self, *package_names_or_classes, **filters):
         self.names = []
         filenames = filters.pop('has_files', [])
+        no_filenames = filters.pop('no_files', [])
         excluded = filters.pop('exclude', [])
         if filters:
             raise ValueError('filter not supported')
 
         def included_in_filter(pkg):
             if pkg.name in excluded:
                 return False
-            return all(pkg.has_file(filename) for filename in filenames)
+            if not all(pkg.has_file(filename) for filename in filenames):
+                return False
+            return not any(pkg.has_file(filename) for filename in no_filenames)
+
         for package_name_or_class in package_names_or_classes:
             if package_name_or_class == ':all:':
                 self._init_all(predicate=included_in_filter)
@@ -41,13 +45,11 @@ def included_in_filter(pkg):
                 self._init_optional(predicate=included_in_filter)
             elif package_name_or_class == ':experimental:':
                 self._init_experimental(predicate=included_in_filter)
-            elif package_name_or_class == ':huge:':
-                self._init_huge(predicate=included_in_filter)
             else:
                 if ':' in package_name_or_class:
                     raise ValueError('a colon may only appear in designators of package types, '
                                      'which must be one of '
-                                     ':all:, :standard:, :optional:, :experimental:, or :huge:, '
+                                     ':all:, :standard:, :optional:, or :experimental:'
                                      'got {}'.format(package_name_or_class))
                 self.names.append(package_name_or_class)
 
@@ -63,9 +65,6 @@ def _init_optional(self, predicate):
     def _init_experimental(self, predicate):
         self.names.extend(pkg.name for pkg in Package.all() if pkg.type == 'experimental' and predicate(pkg))
 
-    def _init_huge(self, predicate):
-        self.names.extend(pkg.name for pkg in Package.all() if pkg.type == 'huge' and predicate(pkg))
-
     def apply(self, function, *args, **kwds):
         for package_name in self.names:
             function(package_name, *args, **kwds)

src/doc/en/developer/packaging.rst

@@ -166,6 +166,8 @@ The following are some additional files which can be added:
     |-- distros
     |   |-- platform1.txt
     |   `-- platform2.txt
+    |-- has_nonfree_dependencies
+    |-- huge
     |-- patches
     |   |-- bar.patch
     |   `-- baz.patch
@@ -729,6 +731,26 @@ packages and optional packages should only depend on standard or
 optional packages.
 
 
+.. _section-spkg-tags:
+
+Package tags
+------------
+
+You can mark a package as "huge" by placing an empty file named
+``huge`` in the package directory.  For example, the package
+``polytopes_db_4d`` is a large database whose compressed tarball has a
+size of 9 GB.
+
+For some other packages, we have placed an empty file named
+``has_nonfree_dependencies`` in the package directory. This is to
+indicate that Sage with this package installed cannot be
+redistributed, and also that the package can only be installed after
+installing some other, non-free package.
+
+We use these tags in our continuous integration scripts to filter
+out packages that we cannot or should not test automatically.
+
+
 .. _section-trees:
 
 Where packages are installed
@@ -992,7 +1014,7 @@ In addition to these fields in ``checksums.ini``, the optional field
 ``upstream_url`` holds an URL to the upstream package archive.
 
 The Release Manager uses the information in ``upstream_url`` to
-download the upstream package archvive and to make it available on the
+download the upstream package archive and to make it available on the
 Sage mirrors when a new release is prepared.  On Trac tickets
 upgrading a package, the ticket description should no longer contain
 the upstream URL to avoid duplication of information.