Actually revert

.gitignore

@@ -94,11 +94,11 @@ ENV/
 
 # Custom stuff
 env.sh
-*config.json
+config.json
 .autoenv.zsh
 
 rsa-key
 tags
 singer-check-tap-data
-state*.json
-catalog*.json
+state.json
+catalog.json

setup.py

@@ -10,19 +10,12 @@
     url="http://singer.io",
     classifiers=["Programming Language :: Python :: 3 :: Only"],
     install_requires=[
-        "singer-python==5.10.0",
+        "singer-python==5.4.0",
         "pendulum",
         "ratelimit",
-        "backoff",
-        "requests",
+        "backoff==1.3.2",
+        "requests==2.20.0",
     ],
-    extras_require={
-        'dev': [
-            'pylint',
-            'ipdb',
-            'nose',
-        ]
-    },
     entry_points="""
     [console_scripts]
     tap-typeform=tap_typeform:main

tap_typeform/__init__.py

@@ -1,23 +1,22 @@
 #!/usr/bin/env python3
+
+import os
+import sys
+import json
+
 import singer
 from singer import utils
-from singer.catalog import Catalog, metadata_module as metadata
-from tap_typeform import streams
-from tap_typeform.context import Context
-from tap_typeform.http import Client
-from tap_typeform import schemas
+from singer.catalog import Catalog, CatalogEntry, Schema
+from . import streams
+from .context import Context
+from . import schemas
 
 REQUIRED_CONFIG_KEYS = ["token", "forms", "incremental_range"]
 
 LOGGER = singer.get_logger()
 
 #def check_authorization(atx):
 #    atx.client.get('/settings')
-class FormMistmatchError(Exception):
-    pass
-
-class NoFormsProvidedError(Exception):
-    pass
 
 
 # Some taps do discovery dynamically where the catalog is read in from a
@@ -26,35 +25,31 @@ class NoFormsProvidedError(Exception):
 #  atx in here since the schema is from file but we would use it if we
 #  pulled schema from the API def discover(atx):
 def discover():
-    streams = []
+    catalog = Catalog([])
     for tap_stream_id in schemas.STATIC_SCHEMA_STREAM_IDS:
         #print("tap stream id=",tap_stream_id)
-        key_properties = schemas.PK_FIELDS[tap_stream_id]
-        schema = schemas.load_schema(tap_stream_id)
-        replication_method = schemas.REPLICATION_METHODS[tap_stream_id].get("replication_method")
-        replication_keys = schemas.REPLICATION_METHODS[tap_stream_id].get("replication_keys")
-        meta = metadata.get_standard_metadata(schema=schema,
-                                              key_properties=key_properties,
-                                              replication_method=replication_method,
-                                              valid_replication_keys=replication_keys)
-
-        meta = metadata.to_map(meta)
-
-        if replication_keys:
-            meta = metadata.write(meta, ('properties', replication_keys[0]), 'inclusion', 'automatic')
-
-        meta = metadata.to_list(meta)
-
-        streams.append({
-            'stream': tap_stream_id,
-            'tap_stream_id': tap_stream_id,
-            'key_properties': key_properties,
-            'schema': schema,
-            'metadata': meta,
-            'replication_method': replication_method,
-            'replication_key': replication_keys[0] if replication_keys else None
-        })
-    return Catalog.from_dict({'streams': streams})
+        schema = Schema.from_dict(schemas.load_schema(tap_stream_id))
+        metadata = []
+        for field_name in schema.properties.keys():
+            #print("field name=",field_name)
+            if field_name in schemas.PK_FIELDS[tap_stream_id]:
+                inclusion = 'automatic'
+            else:
+                inclusion = 'available'
+            metadata.append({
+                'metadata': {
+                    'inclusion': inclusion
+                },
+                'breadcrumb': ['properties', field_name]
+            })
+        catalog.streams.append(CatalogEntry(
+            stream=tap_stream_id,
+            tap_stream_id=tap_stream_id,
+            key_properties=schemas.PK_FIELDS[tap_stream_id],
+            schema=schema,
+            metadata=metadata
+        ))
+    return catalog
 
 
 # this is already defined in schemas.py though w/o dependencies.  do we keep this for the sync?
@@ -86,45 +81,17 @@ def sync(atx):
     LOGGER.info('--------------------')
 
 
-def _compare_forms(config_forms, api_forms):
-    return config_forms.difference(api_forms)
-
-
-def _forms_to_list(config, keyword='forms'):
-    """Splits entries into a list and strips out surrounding blank spaces"""
-    return list(map(str.strip, config.get(keyword).split(',')))
-
-
-def validate_form_ids(config):
-    """Validate the form ids passed in the config"""
-    client = Client(config)
-
-    if not config.get('forms'):
-        LOGGER.fatal("No forms were provided in config")
-        raise NoFormsProvidedError
-
-    config_forms = set(_forms_to_list(config))
-    api_forms = {form.get('id') for form in client.get_forms()}
-
-    mismatched_forms = _compare_forms(config_forms, api_forms)
-
-    if len(mismatched_forms) > 0:
-        LOGGER.fatal(f"FormMistmatchError: forms {mismatched_forms} not returned by API")
-        raise FormMistmatchError
-
-
 @utils.handle_top_exception(LOGGER)
 def main():
     args = utils.parse_args(REQUIRED_CONFIG_KEYS)
     atx = Context(args.config, args.state)
     if args.discover:
-        validate_form_ids(args.config)
         # the schema is static from file so we don't need to pass in atx for connection info.
         catalog = discover()
-        catalog.dump()
+        json.dump(catalog.to_dict(), sys.stdout)
     else:
-        atx.catalog = args.catalog \
-            if args.catalog else discover()
+        atx.catalog = Catalog.from_dict(args.properties) \
+            if args.properties else discover()
         sync(atx)
 
 if __name__ == "__main__":