Merge pull request #555 from GSA/no-more-qa-fork-yay

Fix LOTS of QA Things!
GSA · Sep 30, 2022 · fa215a2 · fa215a2
2 parents 9371d81 + 8b591e6
commit fa215a2
Show file tree

Hide file tree

Showing 4 changed files with 85 additions and 79 deletions.
diff --git a/Makefile b/Makefile
@@ -4,21 +4,22 @@ CKAN_HOME := /srv/app
 
 all: build
 
-ci:
-	docker-compose up -d
+# ###############################################
+# Core commands
+# ###############################################
 
 build:
 	docker build -t ghcr.io/gsa/catalog.data.gov:latest ckan/
 	docker build -t ghcr.io/gsa/catalog.data.gov.solr:latest solr/
 	docker build -t ghcr.io/gsa/catalog.data.gov.db:latest postgresql/
 	docker-compose build
 
+ci:
+	docker-compose up -d
+
 clean:
 	docker-compose down -v --remove-orphans
 
-copy-src:
-	docker cp catalog-app_ckan_1:$(CKAN_HOME)/src .
-
 cypress:
 	# Turn on local system, and open cypress in interactive mode
 	# If you haven't remapped localhost > ckan, you should change baseURL to "http://localhost:5000" in `e2e/cypress.json`
@@ -34,20 +35,54 @@ debug:
 	docker-compose build
 	docker-compose run --service-ports ckan
 
+up:
+	docker-compose up $(ARGS)
+
+update-dependencies:
+	docker-compose run --rm -T ckan /app/ckan/freeze-requirements.sh $(shell id -u) $(shell id -g)
+
+# ###############################################
+# Test commands
+# ###############################################
+
 test:
 	# docker build -t ghcr.io/gsa/catalog.data.gov:latest ckan/
 	docker-compose -f docker-compose.yml -f docker-compose.test.yml build
 	docker-compose -f docker-compose.yml -f docker-compose.test.yml up --abort-on-container-exit test
 
 quick-bat-test:
-	# if local environment is already build and running 
+	# if local environment is already build and running
 	docker-compose -f docker-compose.yml -f docker-compose.test.yml up --abort-on-container-exit test
 
-update-dependencies:
-	docker-compose run --rm -T ckan /app/ckan/freeze-requirements.sh $(shell id -u) $(shell id -g)
+test-extensions:
+	# test our extensions
 
-up:
-	docker-compose up
+	# deal with the CKAN path
+	docker-compose exec ckan bash -c "ln -sf $(CKAN_HOME)/src/ckan $(CKAN_HOME)/ckan"
+
+	# full test datajson
+	docker-compose exec ckan bash -c \
+		"cd $(CKAN_HOME)/src/ckanext-datajson && \
+		 nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/datajson/tests --debug=ckanext"
+
+	# full test datagovtheme
+	docker-compose exec ckan bash -c \
+		"cd $(CKAN_HOME)/src/ckanext-datagovtheme && \
+		 nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/datagovtheme/tests --debug=ckanext"
+
+	# full test geodatagov
+	docker-compose exec ckan bash -c \
+		"cd $(CKAN_HOME)/src/ckanext-geodatagov && \
+		 nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/geodatagov/tests --debug=ckanext"
+
+	# full test geodatagov
+	docker-compose exec ckan bash -c \
+		"cd $(CKAN_HOME)/src/ckanext-datagovdatalog && \
+		 nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/datagovdatalog/tests --debug=ckanext"
+
+# ###############################################
+# Helper commands
+# ###############################################
 
 clear-solr-volume:
 	# Destructive
@@ -62,6 +97,9 @@ unlock-solr-volume:
 search-index-rebuild:
 	docker-compose exec ckan /bin/bash -c "ckan search-index rebuild"
 
+copy-src:
+	docker cp catalog-app_ckan_1:$(CKAN_HOME)/src .
+
 test-import-tool:
 	cd tools/harvest_source_import && \
 		pip install pip==20.3.3  && \
@@ -77,66 +115,34 @@ lint-all:
 				 pip install flake8 && \
 				 flake8 . --count --select=E9 --show-source --statistics"
 
-generate-openess-report:
+qa:
+ifeq (${PARAMS}, all)
+	# PARAMS=all make qa
+	docker-compose exec ckan ckan report generate
+else ifeq (${PARAMS}, openness)
+	# PARAMS=openness make qa
 	# generate report at /report/openness
-	docker-compose exec ckan report generate openness
-
-
-update-tracking-info:
-	# https://docs.ckan.org/en/2.8/maintaining/tracking.html
-	docker-compose exec ckan ckan tracking update
-
-rebuild-search-index:
-	docker-compose exec ckan ckan search-index rebuild
-
-update-qa-info:
+	docker-compose exec ckan ckan report generate openness
+else ifeq (${PARAMS}, update)
+	# PARAMS=update make qa
 	# QA is performed when a dataset/resource is archived, or you can run it manually using a ckan command:
 	docker-compose exec ckan ckan qa update
-
-update-archiver-info:
-	docker-compose exec ckan ckan archiver update
-
-generate-all-reports:
-	docker-compose exec ckan ckan report generate
-
-ckan-worker:
+else ifeq (${PARAMS}, archive)
+	# PARAMS=archive make qa
+	# Archive datasets to perform QA
+	docker-compose exec ckan ckan archiver update --queue bulk
+else ifeq (${PARAMS}, worker)
+	# PARAMS=worker make qa
 	docker-compose exec ckan ckan jobs worker bulk
+endif
 
-archiver-worker:
-	export C_FORCE_ROOT=1  # celery don't want to run as root
-	docker-compose exec ckan ckan celeryd2 run all
-
-harvest-fetch-queue:
-	docker-compose exec ckan ckan harvester fetch-consumer
-
-harvest-gather-queue:
-	docker-compose exec ckan ckan harvester gather-consumer
-
-harvest-check-finished-jobs:
-	docker-compose exec ckan ckan harvester run
-
-test-extensions:
-	# test our extensions
-
-	# deal with the CKAN path
-	docker-compose exec ckan bash -c "ln -sf $(CKAN_HOME)/src/ckan $(CKAN_HOME)/ckan"
-
-	# full test datajson
-	docker-compose exec ckan bash -c \
-		"cd $(CKAN_HOME)/src/ckanext-datajson && \
-		 nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/datajson/tests --debug=ckanext"
-
-	# full test datagovtheme
-	docker-compose exec ckan bash -c \
-		"cd $(CKAN_HOME)/src/ckanext-datagovtheme && \
-		 nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/datagovtheme/tests --debug=ckanext"
-
-	# full test geodatagov
-	docker-compose exec ckan bash -c \
-		"cd $(CKAN_HOME)/src/ckanext-geodatagov && \
-		 nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/geodatagov/tests --debug=ckanext"
+update-tracking-info:
+	# https://docs.ckan.org/en/2.8/maintaining/tracking.html
+	docker-compose exec ckan ckan tracking update
 
-	# full test geodatagov
-	docker-compose exec ckan bash -c \
-		"cd $(CKAN_HOME)/src/ckanext-datagovdatalog && \
-		 nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/datagovdatalog/tests --debug=ckanext"
+harvest:
+	# Pass any of the following arguments to run them
+	# make harvest 'fetch-consumer'
+	# make harvest 'gather-consumer'
+	# make harvest 'run'
+	docker-compose exec ckan ckan harvester $(ARGS)
diff --git a/README.md b/README.md
@@ -233,7 +233,7 @@ pdb; pdb.set_trace()`) you will be able to inspect it in this terminal next time
 the code is executed.
 If you are testing a harvest process (gather/fetch/run), try turning off the command
 to start in the background in the `ckan/docker-entrypoint.d/10-setup-harvest.sh`.
-Then, run the relevant command manually (`make harvest-fetch-queue`) after startup.
+Then, run the relevant command manually (`make harvest fetch-queue`) after startup.
 
 ## SAML2
 

diff --git a/ckan/requirements.in b/ckan/requirements.in
@@ -6,7 +6,7 @@ ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@master
 -e git+https://github.com/gsa/ckanext-spatial.git@datagov#egg=ckanext-spatial
 # git+https://github.com/keitaroinc/ckanext-saml2auth.git@main#egg=ckanext-saml2auth
 git+https://github.com/gsa/ckanext-saml2auth.git@main#egg=ckanext-saml2auth
--e git+https://github.com/gsa/ckanext-qa.git@datagov#egg=ckanext-qa
+-e git+https://github.com/ckan/ckanext-qa.git@master#egg=ckanext-qa
 -e git+https://github.com/ckan/ckanext-archiver.git@master#egg=ckanext-archiver
 -e git+https://github.com/ckan/ckanext-report.git@master#egg=ckanext-report
 

diff --git a/ckan/requirements.txt b/ckan/requirements.txt
@@ -3,22 +3,22 @@ async-timeout==4.0.2
 Babel==2.9.1
 Beaker==1.11.0
 bleach==3.3.0
-boto3==1.24.76
-botocore==1.27.76
-certifi==2022.9.14
+boto3==1.24.83
+botocore==1.27.83
+certifi==2022.9.24
 cffi==1.15.1
 chardet==3.0.4
 ckan @ git+https://github.com/ckan/ckan.git@65af260bef26ea99981597f872d39da0a3f4f3f9
--e git+https://github.com/ckan/ckanext-archiver.git@534b6db93be81e866ac282e4ea10b89c3563a0e8#egg=ckanext_archiver
+-e git+https://github.com/ckan/ckanext-archiver.git@c96e3c81bfc430cdb0372f3307c7abd4109a80f1#egg=ckanext_archiver
 ckanext-datagovcatalog==0.0.4
 ckanext-datagovtheme==0.1.8
 ckanext-datajson==0.1.3
-ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@c595ec71f5c39112a0769d62669d7f633c981331
+ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@6fb70aaa00ca9e790535e12f96c50f5c39017f6e
 ckanext-envvars==0.0.2
 ckanext-geodatagov==0.1.7
 ckanext-googleanalyticsbasic==0.2.0
 -e git+https://github.com/ckan/ckanext-harvest.git@cb0a7034410f217b2274585cb61783582832c8d5#egg=ckanext_harvest
--e git+https://github.com/gsa/ckanext-qa.git@c99b9f52a3844d862d191989bfc363ae7695569e#egg=ckanext_qa
+-e git+https://github.com/ckan/ckanext-qa.git@1731b59d2bf82b06f7866c204b26eb7c6c9ea1f9#egg=ckanext_qa
 -e git+https://github.com/ckan/ckanext-report.git@24ade58d126d9480286c7bd1a904f20186d83988#egg=ckanext_report
 ckanext-saml2auth @ git+https://github.com/gsa/ckanext-saml2auth.git@0493d1d7ad321760911bed8b189ecbf02099033d
 -e git+https://github.com/gsa/ckanext-spatial.git@fbd3e2b3f67cc1038e4c7950731438b36da4d8b1#egg=ckanext_spatial
@@ -50,11 +50,11 @@ jmespath==1.0.1
 json-table-schema==0.2.1
 jsonschema==2.4.0
 lxml==4.9.1
-Mako==1.2.2
+Mako==1.2.3
 Markdown==3.1.1
 MarkupSafe==2.1.1
 messytables==0.15.2
-newrelic==8.1.0.180
+newrelic==8.2.0.181
 nose==1.3.7
 OWSLib==0.18.0
 packaging==21.3
@@ -70,7 +70,7 @@ progressbar2==3.53.3
 psycopg2==2.8.6
 pycparser==2.21
 PyJWT==2.4.0
-pyOpenSSL==22.0.0
+pyOpenSSL==22.1.0
 pyparsing==3.0.9
 pyproj==2.6.1
 pysaml2==7.0.1
@@ -109,7 +109,7 @@ Werkzeug==2.0.0
 wheel==0.37.1
 wrapt==1.14.1
 xlrd==2.0.1
-xmlschema==2.0.4
+xmlschema==2.1.0
 zipp==3.8.1
 zope.event==4.5.0
 zope.interface==5.4.0