Merge branch 'issue-36' into issue-38

.gitignore

@@ -1,2 +1,3 @@
 __pycache__
-dist
+dist
+example/test.ini

src/nsidc/metgen/cli.py

@@ -25,20 +25,22 @@ def init(config):
 def info(config_filename):
     """Summarizes the contents of a configuration file."""
     click.echo(metgen.banner())
-    configuration = config.configuration(config.config_parser(config_filename), {})
+    configuration = config.configuration(config.config_parser_factory(config_filename), {})
     configuration.show()
 
 @cli.command()
 @click.option('-c', '--config', 'config_filename', help='Path to configuration file', required=True)
 @click.option('-e', '--env', help='environment', default=constants.DEFAULT_CUMULUS_ENVIRONMENT, show_default=True)
+@click.option('-n', '--number', help="Process at most 'count' granules.", metavar='count', required=False, default=-1)
 @click.option('-wc', '--write-cnm', is_flag=True, help="Write CNM messages to files.")
-def process(config_filename, env, write_cnm):
+def process(config_filename, env, write_cnm, number):
     """Processes science data files based on configuration file contents."""
     click.echo(metgen.banner())
     overrides = {
-        'write_cnm_file': write_cnm
+        'write_cnm_file': write_cnm,
+        'number': number
     }
-    configuration = config.configuration(config.config_parser(config_filename), overrides, env)
+    configuration = config.configuration(config.config_parser_factory(config_filename), overrides, env)
     try:
         metgen.process(configuration)
     except Exception as e:

src/nsidc/metgen/config.py

@@ -20,6 +20,7 @@ class Config:
     kinesis_arn: str
     write_cnm_file: bool
     checksum_type: str
+    number: int
 
     def show(self):
         # TODO add section headings in the right spot (if we think we need them in the output)
@@ -50,33 +51,39 @@ def collection_from_cmr(self, mapping):
             'version': mapping['version']
         }
 
-def config_parser(configuration_file):
+def config_parser_factory(configuration_file):
     if configuration_file is None or not os.path.exists(configuration_file):
         raise ValueError(f'Unable to find configuration file {configuration_file}')
     cfg_parser = configparser.ConfigParser()
     cfg_parser.read(configuration_file)
     return cfg_parser
 
 
-def _get_configuration_value(section, name, config_parser, overrides, default=None):
+def _get_configuration_value(section, name, value_type, config_parser, overrides, default=None):
     if overrides.get(name) is None:
-        return config_parser.get(section, name, fallback=default)
+        if value_type is bool:
+            return config_parser.getboolean(section, name, fallback=default)
+        elif value_type is int:
+            return config_parser.getint(section, name, fallback=default)
+        else:
+            return config_parser.get(section, name, fallback=default)
     else:
         return overrides.get(name)
 
 def configuration(config_parser, overrides, environment=constants.DEFAULT_CUMULUS_ENVIRONMENT):
     try:
         return Config(
             environment,
-            _get_configuration_value('Source', 'data_dir', config_parser, overrides),
-            _get_configuration_value('Collection', 'auth_id', config_parser, overrides),
-            _get_configuration_value('Collection', 'version', config_parser, overrides),
-            _get_configuration_value('Collection', 'provider', config_parser, overrides),
-            _get_configuration_value('Destination', 'local_output_dir', config_parser, overrides),
-            _get_configuration_value('Destination', 'ummg_dir', config_parser, overrides),
-            _get_configuration_value('Destination', 'kinesis_arn', config_parser, overrides),
-            _get_configuration_value('Destination', 'write_cnm_file', config_parser, overrides, False),
-            _get_configuration_value('Settings', 'checksum_type', config_parser, overrides, 'SHA256'),
+            _get_configuration_value('Source', 'data_dir', str, config_parser, overrides),
+            _get_configuration_value('Collection', 'auth_id', str, config_parser, overrides),
+            _get_configuration_value('Collection', 'version', int, config_parser, overrides),
+            _get_configuration_value('Collection', 'provider', str, config_parser, overrides),
+            _get_configuration_value('Destination', 'local_output_dir', str, config_parser, overrides),
+            _get_configuration_value('Destination', 'ummg_dir', str, config_parser, overrides),
+            _get_configuration_value('Destination', 'kinesis_arn', str, config_parser, overrides),
+            _get_configuration_value('Destination', 'write_cnm_file', bool, config_parser, overrides, False),
+            _get_configuration_value('Settings', 'checksum_type', str, config_parser, overrides, 'SHA256'),
+            _get_configuration_value('Settings', 'number', int, config_parser, overrides, -1),
         )
     except Exception as e:
         return Exception('Unable to read the configuration file', e)

src/nsidc/metgen/metgen.py

@@ -16,6 +16,7 @@
 SOURCE_SECTION_NAME = 'Source'
 COLLECTION_SECTION_NAME = 'Collection'
 DESTINATION_SECTION_NAME = 'Destination'
+SETTINGS_SECTION_NAME = 'Settings'
 UMMG_BODY_TEMPLATE = 'src/nsidc/metgen/templates/ummg_body_template.json'
 UMMG_TEMPORAL_TEMPLATE = 'src/nsidc/metgen/templates/ummg_temporal_single_template.json'
 UMMG_SPATIAL_TEMPLATE = 'src/nsidc/metgen/templates/ummg_horizontal_rectangle_template.json'
@@ -72,13 +73,13 @@ def init_config(configuration_file):
     cfg_parser.set(DESTINATION_SECTION_NAME, "local_output_dir", Prompt.ask("Local output directory", default="output"))
     cfg_parser.set(DESTINATION_SECTION_NAME, "ummg_dir", Prompt.ask("Local UMM-G output directory (relative to local output directory)", default="ummg"))
     cfg_parser.set(DESTINATION_SECTION_NAME, "kinesis_arn", Prompt.ask("Kinesis Stream ARN"))
-    cfg_parser.set("Destination", "write_cnm_file", Prompt.ask("Write CNM messages to files (True/False)"))
+    cfg_parser.set(DESTINATION_SECTION_NAME, "write_cnm_file", Prompt.ask("Write CNM messages to files (True/False)"))
 
     print()
-    print("Settings Parameters")
+    print(f'{SETTINGS_SECTION_NAME} Parameters')
     print('--------------------------------------------------')
-    cfg_parser.add_section("Settings")
-    cfg_parser.set("Settings", "checksum_type", Prompt.ask("Checksum type", default="SHA256"))
+    cfg_parser.add_section(SETTINGS_SECTION_NAME)
+    cfg_parser.set(SETTINGS_SECTION_NAME, "checksum_type", Prompt.ask("Checksum type", default="SHA256"))
 
     print()
     print(f'Saving new configuration: {configuration_file}')
@@ -87,10 +88,6 @@ def init_config(configuration_file):
 
     return configuration_file
 
-def read_config(configuration):
-    mapping = dict(configuration.__dict__)
-    return mapping
-
 def show_config(configuration):
     # TODO add section headings in the right spot (if we think we need them in the output)
     print()
@@ -119,6 +116,9 @@ def process(configuration):
 
     granules = granule_paths(Path(configuration.data_dir))
     print(f'Found {len(granules.items())} granules to process')
+    if (configuration.number > 0 and configuration.number < len(granules)):
+        print(f'Processing the first {configuration.number} granule(s)')
+        granules = granules[:configuration.number]
     print()
 
     # TODO: create local_output_dir, ummg_dir, and cnm subdir if they don't exist

tests/test_cli.py

@@ -6,6 +6,14 @@
 from nsidc.metgen.cli import cli
 
 
+# Unit tests for the 'cli' module functions.
+#
+# The test boundary is the cli module's interface with the metgen module, so in
+# addition to testing the cli module's behavior, the tests should mock that
+# module's functions and assert that cli functions call them with the correct
+# parameters, correctly handle their return values, and handle any exceptions
+# they may throw.
+
 @pytest.fixture
 def cli_runner():
     return CliRunner()
@@ -45,4 +53,20 @@ def test_process_requires_config_does_not_call_process(mock, cli_runner):
 @patch('nsidc.metgen.metgen.process')
 def test_process_with_config_calls_process(mock, cli_runner):
     result = cli_runner.invoke(cli, ['process', '--config', './example/modscg.ini'])
-    assert mock.called
+    assert mock.called
+
+@patch('nsidc.metgen.metgen.process')
+def test_process_with_granule_limit(process_mock, cli_runner):
+    number_files = 2
+    result = cli_runner.invoke(cli, ['process', '-n', str(number_files), '--config', './example/modscg.ini'])
+
+    assert process_mock.called
+    args = process_mock.call_args.args
+    assert len(args) == 1
+    configuration = args[0]
+    assert configuration.number == number_files
+    assert result.exit_code == 0
+
+
+# TODO: When process raises an exception, cli handles it and displays a message
+#       and has non-zero exit code

tests/test_config.py

@@ -8,6 +8,28 @@
 from nsidc.metgen import metgen
 
 
+# Unit tests for the 'config' module functions.
+#
+# The test boundary is the config module's interface with the filesystem and
+# the aws module, so in addition to testing the config module's behavior, the
+# tests should mock those module's functions and assert that config functions
+# call them with the correct parameters, correctly handle their return values,
+# and handle any exceptions they may throw.
+
+@pytest.fixture
+def expected_keys():
+    return set(['environment',
+                'data_dir',
+                'auth_id',
+                'version',
+                'provider',
+                'local_output_dir',
+                'ummg_dir',
+                'kinesis_arn',
+                'write_cnm_file',
+                'checksum_type',
+                'number'])
+
 @pytest.fixture
 def cfg_parser():
     cp = ConfigParser()
@@ -22,35 +44,39 @@ def cfg_parser():
     cp['Destination'] = {
         'local_output_dir': '/output/here',
         'ummg_dir': 'ummg',
-        'kinesis_arn': 'abcd-1234'
+        'kinesis_arn': 'abcd-1234',
+        'write_cnm_file': False
     }
     return cp
 
 
 def test_config_parser_without_filename():
     with pytest.raises(ValueError):
-        config.config_parser(None)
+        config.config_parser_factory(None)
 
 @patch('nsidc.metgen.metgen.os.path.exists', return_value = True)
 def test_config_parser_return_type(mock):
-    result = config.config_parser('foo.ini')
+    result = config.config_parser_factory('foo.ini')
     assert isinstance(result, ConfigParser)
 
 def test_config_from_config_parser(cfg_parser):
     cfg = config.configuration(cfg_parser, {}, constants.DEFAULT_CUMULUS_ENVIRONMENT)
     assert isinstance(cfg, config.Config)
 
-def test_config_with_values(cfg_parser):
-    expected_keys = set(['environment',
-                         'data_dir',
-                         'auth_id',
-                         'version',
-                         'provider',
-                         'local_output_dir',
-                         'ummg_dir',
-                         'kinesis_arn',
-                         'write_cnm_file',
-                         'checksum_type'])
+def test_config_with_no_write_cnm(cfg_parser, expected_keys):
+    cfg = config.configuration(cfg_parser, {})
+
+    config_keys = set(cfg.__dict__)
+    assert len(config_keys - expected_keys) == 0
+
+    assert cfg.data_dir == '/data/example'
+    assert cfg.auth_id == 'DATA-0001'
+    assert cfg.kinesis_arn == 'abcd-1234'
+    assert cfg.environment == 'uat'
+    assert not cfg.write_cnm_file
+
+def test_config_with_write_cnm(cfg_parser, expected_keys):
+    cfg_parser.set("Destination", "write_cnm_file", 'True')
     cfg = config.configuration(cfg_parser, {})
 
     config_keys = set(cfg.__dict__)
@@ -60,30 +86,32 @@ def test_config_with_values(cfg_parser):
     assert cfg.auth_id == 'DATA-0001'
     assert cfg.kinesis_arn == 'abcd-1234'
     assert cfg.environment == 'uat'
+    assert cfg.write_cnm_file == True
 
 def test_enhanced_config():
     myconfig = config.Config('env', 'data_dir', 'auth_id', 'version',
-                  'provider', 'output_dir', 'ummg_dir', 'arn', 'write_cnm_file', 'checksum_type')
+                  'provider', 'output_dir', 'ummg_dir', 'arn',
+                  'write_cnm_file', 'checksum_type', 'number')
     enhanced_config = myconfig.enhance('pgid')
     assert set(['auth_id', 'version', 'producer_granule_id',
                 'submission_time', 'uuid']) <= set(enhanced_config.keys())
 
 def test_get_configuration_value(cfg_parser):
-    result = config._get_configuration_value("Source", "data_dir", cfg_parser, {})
+    result = config._get_configuration_value("Source", "data_dir", str, cfg_parser, {})
     assert result == cfg_parser.get("Source", "data_dir")
 
 def test_get_configuration_value_with_override(cfg_parser):
     overrides = { 'data_dir': 'foobar' }
-    result = config._get_configuration_value("Source", "data_dir", cfg_parser, overrides)
+    result = config._get_configuration_value("Source", "data_dir", str, cfg_parser, overrides)
     assert result == overrides['data_dir']
 
 def test_get_configuration_value_with_default(cfg_parser):
     default_value = '/etc/foobar'
-    result = config._get_configuration_value("Source", "foobar_dir", cfg_parser, {}, default_value)
+    result = config._get_configuration_value("Source", "foobar_dir", str, cfg_parser, {}, default_value)
     assert result == default_value
 
 def test_get_configuration_value_with_default_and_override(cfg_parser):
     overrides = { 'data_dir': 'foobar' }
     default_value = '/etc/foobar'
-    result = config._get_configuration_value("Source", "data_dir", cfg_parser, overrides, default_value)
+    result = config._get_configuration_value("Source", "data_dir", str, cfg_parser, overrides, default_value)
     assert result == overrides['data_dir']

tests/test_metgen.py

@@ -7,6 +7,13 @@
 from nsidc.metgen import constants
 from nsidc.metgen import metgen
 
+# Unit tests for the 'metgen' module functions.
+#
+# The test boundary is the metgen module's interface with the filesystem and
+# the aws & config modules, so in addition to testing the metgen module's
+# behavior, the tests should mock those module's functions and assert that
+# metgen functions call them with the correct parameters, correctly handle
+# their return values, and handle any exceptions they may throw.
 
 @pytest.fixture
 def cfg_parser():
@@ -29,9 +36,6 @@ def cfg_parser():
 def test_banner():
     assert len(metgen.banner()) > 0
 
-def test_read_config(cfg_parser):
-    mapping = metgen.read_config(config.configuration(cfg_parser, {}))
-
 def test_sums_file_sizes():
     details = {
         'first_id': {
@@ -52,3 +56,7 @@ def test_sums_file_sizes():
     assert summary['production_date_time'] == 'then'
     assert summary['date_time'] == 'now'
     assert summary['geometry'] == 'big'
+
+# TODO: Test that it writes files if 'write cnm' flag is True
+
+# TODO: Test that it does not write files if 'write cnm' flag is False