From 1a122d4e93bf5e55307445774d2a1c64e9e2828e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 23 Oct 2020 18:44:05 +0200 Subject: [PATCH 1/6] Remove reference_data.sql usages. #7256 --- scripts/installer/Makefile | 9 +---- scripts/installer/README.txt | 4 -- scripts/installer/install | 71 +----------------------------------- scripts/installer/install.py | 27 -------------- 4 files changed, 3 insertions(+), 108 deletions(-) diff --git a/scripts/installer/Makefile b/scripts/installer/Makefile index 7e002e8fa61..5155edfde1f 100644 --- a/scripts/installer/Makefile +++ b/scripts/installer/Makefile @@ -3,7 +3,6 @@ DISTRIBUTION_WAR_FILE=${INSTALLER_ZIP_DIR}/dataverse.war GLASSFISH_SETUP_SCRIPT=${INSTALLER_ZIP_DIR}/as-setup.sh POSTGRES_DRIVERS=${INSTALLER_ZIP_DIR}/pgdriver API_SCRIPTS=${INSTALLER_ZIP_DIR}/setup-datasetfields.sh ${INSTALLER_ZIP_DIR}/setup-users.sh ${INSTALLER_ZIP_DIR}/setup-builtin-roles.sh ${INSTALLER_ZIP_DIR}/setup-dvs.sh ${INSTALLER_ZIP_DIR}/data ${INSTALLER_ZIP_DIR}/setup-identity-providers.sh ${INSTALLER_ZIP_DIR}/setup-all.sh ${INSTALLER_ZIP_DIR}/post-install-api-block.sh -DB_SCRIPT=${INSTALLER_ZIP_DIR}/reference_data.sql JHOVE_CONFIG=${INSTALLER_ZIP_DIR}/jhove.conf JHOVE_SCHEMA=${INSTALLER_ZIP_DIR}/jhoveConfig.xsd SOLR_SCHEMA=${INSTALLER_ZIP_DIR}/schema.xml ${INSTALLER_ZIP_DIR}/schema_dv_mdb_fields.xml ${INSTALLER_ZIP_DIR}/schema_dv_mdb_copies.xml ${INSTALLER_ZIP_DIR}/updateSchemaMDB.sh @@ -16,9 +15,9 @@ installer: dvinstall.zip clean: /bin/rm -rf ${INSTALLER_ZIP_DIR} dvinstall.zip -dvinstall.zip: ${GLASSFISH_SETUP_SCRIPT} ${POSTGRES_DRIVERS} ${JSF_PATCH_DIR} ${DISTRIBUTION_WAR_FILE} ${API_SCRIPTS} ${DB_SCRIPT} ${JHOVE_CONFIG} ${JHOVE_SCHEMA} ${SOLR_SCHEMA} ${SOLR_CONFIG} ${PYTHON_FILES} ${INSTALL_SCRIPT} +dvinstall.zip: ${GLASSFISH_SETUP_SCRIPT} ${POSTGRES_DRIVERS} ${JSF_PATCH_DIR} ${DISTRIBUTION_WAR_FILE} ${API_SCRIPTS} ${JHOVE_CONFIG} ${JHOVE_SCHEMA} ${SOLR_SCHEMA} ${SOLR_CONFIG} ${PYTHON_FILES} ${INSTALL_SCRIPT} @echo making installer... - zip -r dvinstall.zip ${GLASSFISH_SETUP_SCRIPT} ${POSTGRES_DRIVERS} ${JSF_PATCH_DIR} ${DISTRIBUTION_WAR_FILE} ${API_SCRIPTS} ${DB_SCRIPT} ${JHOVE_CONFIG} ${JHOVE_SCHEMA} ${SOLR_SCHEMA} ${SOLR_CONFIG} ${PYTHON_FILES} ${INSTALL_SCRIPT} + zip -r dvinstall.zip ${GLASSFISH_SETUP_SCRIPT} ${POSTGRES_DRIVERS} ${JSF_PATCH_DIR} ${DISTRIBUTION_WAR_FILE} ${API_SCRIPTS} ${JHOVE_CONFIG} ${JHOVE_SCHEMA} ${SOLR_SCHEMA} ${SOLR_CONFIG} ${PYTHON_FILES} ${INSTALL_SCRIPT} @echo @echo "Done!" @@ -55,10 +54,6 @@ ${API_SCRIPTS}: ../api/setup-datasetfields.sh ../api/setup-users.sh ../api/setup @echo copying api scripts /bin/cp -R ../api/setup-datasetfields.sh ../api/setup-users.sh ../api/setup-dvs.sh ../api/setup-identity-providers.sh ../api/setup-all.sh ../api/post-install-api-block.sh ../api/setup-builtin-roles.sh ../api/data ${INSTALLER_ZIP_DIR} -${DB_SCRIPT}: ../database/reference_data.sql ${INSTALLER_ZIP_DIR} - @echo copying reference data sql script - /bin/cp ../database/reference_data.sql ${INSTALLER_ZIP_DIR} - ${JHOVE_CONFIG}: ../../conf/jhove/jhove.conf ${INSTALLER_ZIP_DIR} @echo copying jhove config file /bin/cp ../../conf/jhove/jhove.conf ${INSTALLER_ZIP_DIR} diff --git a/scripts/installer/README.txt b/scripts/installer/README.txt index 81328b89ccd..ec11881c720 100644 --- a/scripts/installer/README.txt +++ b/scripts/installer/README.txt @@ -32,10 +32,6 @@ setup-identity-providers.sh setup-users.sh data (the entire directory with all its contents) -from scripts/database: - -reference_data.sql - from conf/jhove: jhove.conf diff --git a/scripts/installer/install b/scripts/installer/install index c804e1903e4..ffe98d48720 100755 --- a/scripts/installer/install +++ b/scripts/installer/install @@ -46,8 +46,7 @@ if (exists($ENV{'MY_POD_NAME'})) my $jq_exec_path = ""; my $psql_exec_path = ""; my $cwd; -my $WARFILE_LOCATION = "dataverse.war"; -my $REFERENCE_DATA_SQL = "../database/reference_data.sql"; +my $WARFILE_LOCATION = "dataverse.war"; my @CONFIG_VARIABLES; @@ -317,23 +316,6 @@ unless ($postgresonly) # 1b. check and remember the working dir: chomp( $cwd = `pwd` ); -# 1c. check if the reference data SQL file is there: - - $REFERENCE_DATA_SQL = "../database/reference_data.sql"; - - unless ( -f $REFERENCE_DATA_SQL ) { - # if it's not in ../database, then we're probably running out of the - # unzipped installer bundle, so it should be right here in the current directory: - $REFERENCE_DATA_SQL = "reference_data.sql"; - } - - unless ( -f $REFERENCE_DATA_SQL ) { - print "\nWARNING: Can't find .sql data template!\n"; - print "(are you running the installer in the right directory?)\n"; - - exit 0; - } - # 1d. jq executable: my $sys_path = $ENV{'PATH'}; @@ -563,15 +545,6 @@ my $glassfish_dir = $CONFIG_DEFAULTS{'GLASSFISH_DIRECTORY'}; my $done = &setup_appserver(); -# 7. POPULATE THE DATABASE WITH REFERENCE DATA -# TODO: move this further down -# (in this step some pre-supplied content is inserted into the database that we have just created; -# it is not *necessary* for the application to run in the very basic mode; but some features - certain -# types of metadata imports, for example - will be unavailable if it's not done. - -&import_reference_data(); - - # Check if the App is running: unless (( @@ -1513,48 +1486,6 @@ sub setup_postgres { } } -sub import_reference_data { - print "\npopulating the database with reference data:\n\n"; - - # (we have already verified that the referenceData.sql file exists) - - my $psql_command = $psql_exec_path . "/psql -h " . $CONFIG_DEFAULTS{'POSTGRES_SERVER'} - . " -p " . $CONFIG_DEFAULTS{'POSTGRES_PORT'} - . " -U " . $CONFIG_DEFAULTS{'POSTGRES_USER'} - . " -d " . $CONFIG_DEFAULTS{'POSTGRES_DATABASE'} . " -f " . $REFERENCE_DATA_SQL; - - print "EXECUTING PSQL COMMAND: $psql_command\n"; - unless ( ( my $exitcode = system("$psql_command") ) == 0 ) - { - print "WARNING: Could not pre-populate Postgres database for the Dataverse application!\n"; - print "(command: " . $psql_command . ")\n"; - print "(psql exit code: " . $exitcode . ")\n"; - print "\nYou must populate the database in order for all the features of your \n"; - print "new Dataverse to be available. \n"; - print "\n"; - print "You can try this again, by executing the following on the command line:\n"; - print " psql -U $CONFIG_DEFAULTS{'POSTGRES_USER'} -d $CONFIG_DEFAULTS{'POSTGRES_DATABASE'} -f $REFERENCE_DATA_SQL\n"; - print "then re-start payara \n\n"; - print "\n"; - print "If it's still failing, please consult the installation manual and/or\n"; - print "seek support from the Dataverse team.\n\n"; - - print "Press any key to continue... "; - - unless ($noninteractive) - { - system "stty cbreak /dev/tty 2>&1"; - my $key = getc(STDIN); - system "stty -cbreak /dev/tty 2>&1"; - print "\n"; - } - } - else - { - print "\nOK, done!\n"; - } -} - sub read_config_defaults { my $config_file = shift @_; diff --git a/scripts/installer/install.py b/scripts/installer/install.py index a00250a44ce..40f91cea640 100644 --- a/scripts/installer/install.py +++ b/scripts/installer/install.py @@ -245,17 +245,6 @@ sys.exit("Sorry, I can't seem to find an appropriate warfile.\nAre you running the installer from the right directory?") print(warfile+" available to deploy. Good.") - # 1b. check for reference_data.sql - referenceData = '../database/reference_data.sql' - if not os.path.isfile(referenceData): - # if it's not there, then we're probably running out of the - # unzipped installer bundle, so it should be right here in the current directory: - referenceData = 'reference_data.sql' - if not os.path.isfile(referenceData): - sys.exit("Can't find reference_data.sql!\nAre you running the installer from the right directory?") - - print("found "+referenceData+"... good") - # 1c. check if jq is available # (but we're only doing it if it's not that weird "pod name" mode) if podName != "start-glassfish": @@ -558,22 +547,6 @@ # @todo: restart/try to deploy again if it failed? # @todo: if asadmin deploy says it was successful, verify that the application is running... if not - repeat the above? -# 6. Import reference data -print("importing reference data...") -# open the new postgresQL connection (as the application user): -conn_string="dbname='"+pgDb+"' user='"+pgUser+"' password='"+pgPassword+"' host='"+pgHost+"'" -conn = psycopg2.connect(conn_string) -conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) -cur = conn.cursor() -try: - cur.execute(open(referenceData, "r").read()) - print("done.") -except: - print("WARNING: failed to import reference data!") - -cur.close() -conn.close() - # 7. RUN SETUP SCRIPTS AND CONFIGURE EXTRA SETTINGS # (note that we may need to change directories, depending on whether this is a dev., or release installer) # 7a. run setup scripts From fc3dbb690778f1b3c3dc70bb0e34abec22b3a12b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 23 Oct 2020 20:27:04 +0200 Subject: [PATCH 2/6] Move index creation from reference_data.sql into Flyway baseline. #7256 --- scripts/database/reference_data.sql | 13 ------------- .../db/migration/V1__flyway_schema_baseline.sql | 11 +++++++++++ 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/scripts/database/reference_data.sql b/scripts/database/reference_data.sql index 15ddedd4301..1bc6bcd4bab 100644 --- a/scripts/database/reference_data.sql +++ b/scripts/database/reference_data.sql @@ -26,19 +26,6 @@ INSERT INTO guestbook( VALUES ( false, true, false, now(), 'Default', false, false, null); --- TODO: Remove if http://stackoverflow.com/questions/25743191/how-to-add-a-case-insensitive-jpa-unique-constraint --- gets an answer. See also https://github.com/IQSS/dataverse/issues/2598#issuecomment-158219334 -CREATE UNIQUE INDEX dataverse_alias_unique_idx on dataverse (LOWER(alias)); -CREATE UNIQUE INDEX index_authenticateduser_lower_email ON authenticateduser (lower(email)); --- CREATE UNIQUE INDEX index_authenticateduser_lower_useridentifier ON authenticateduser (lower(useridentifier)); --- this field has been removed from builtinuser; CREATE UNIQUE INDEX index_builtinuser_lower_email ON builtinuser (lower(email)); - ---Edit Dataset: Investigate and correct multiple draft issue: https://github.com/IQSS/dataverse/issues/2132 ---This unique index will prevent the multiple draft issue -CREATE UNIQUE INDEX one_draft_version_per_dataset ON datasetversion -(dataset_id) WHERE versionstate='DRAFT'; - - INSERT INTO worldmapauth_tokentype ( name, created, diff --git a/src/main/resources/db/migration/V1__flyway_schema_baseline.sql b/src/main/resources/db/migration/V1__flyway_schema_baseline.sql index e69de29bb2d..2ec219cd19e 100644 --- a/src/main/resources/db/migration/V1__flyway_schema_baseline.sql +++ b/src/main/resources/db/migration/V1__flyway_schema_baseline.sql @@ -0,0 +1,11 @@ +-- TODO: we still should add the real base line here, too. That would avoid conflicts between EclipseLink +-- trying to create new tables on existing databases. See https://github.com/IQSS/dataverse/issues/5871 + +-- This is unsupported by JPA, as it is PostgreSQL specific. Has to be done here, cannot be done in code. +-- (Only other option would be a lowercase copy of the data as a separate column, automatically filled py JPA) +CREATE UNIQUE INDEX IF NOT EXISTS dataverse_alias_unique_idx on dataverse (LOWER(alias)); +CREATE UNIQUE INDEX IF NOT EXISTS index_authenticateduser_lower_email ON authenticateduser (lower(email)); + +-- Edit Dataset: Investigate and correct multiple draft issue: https://github.com/IQSS/dataverse/issues/2132 +-- This unique index will prevent the multiple draft issue, yet it cannot be done in JPA code. +CREATE UNIQUE INDEX IF NOT EXISTS one_draft_version_per_dataset ON datasetversion (dataset_id) WHERE versionstate='DRAFT'; \ No newline at end of file From 4611825c2f2a01088f9fb56991e085ff7cbf3831 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 23 Oct 2020 21:01:17 +0200 Subject: [PATCH 3/6] Replace initial data insert from reference_data.sql with Flyway afterMigrate callback SQL script. #7256 --- scripts/database/reference_data.sql | 39 ----------------- .../afterMigrate__1-upsert-referenceData.sql | 42 +++++++++++++++++++ 2 files changed, 42 insertions(+), 39 deletions(-) delete mode 100644 scripts/database/reference_data.sql create mode 100644 src/main/resources/db/migration/afterMigrate__1-upsert-referenceData.sql diff --git a/scripts/database/reference_data.sql b/scripts/database/reference_data.sql deleted file mode 100644 index 1bc6bcd4bab..00000000000 --- a/scripts/database/reference_data.sql +++ /dev/null @@ -1,39 +0,0 @@ --- using http://dublincore.org/schemas/xmls/qdc/dcterms.xsd because at http://dublincore.org/schemas/xmls/ it's the schema location for http://purl.org/dc/terms/ which is referenced in http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html -INSERT INTO foreignmetadataformatmapping(id, name, startelement, displayName, schemalocation) VALUES (1, 'http://purl.org/dc/terms/', 'entry', 'dcterms: DCMI Metadata Terms', 'http://dublincore.org/schemas/xmls/qdc/dcterms.xsd'); -INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) VALUES (1, ':title', 'title', FALSE, NULL, 1 ); -INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) VALUES (2, ':identifier', 'otherIdValue', FALSE, NULL, 1 ); -INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) VALUES (3, ':creator', 'authorName', FALSE, NULL, 1 ); -INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) VALUES (4, ':date', 'productionDate', FALSE, NULL, 1 ); -INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) VALUES (5, ':subject', 'keywordValue', FALSE, NULL, 1 ); -INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) VALUES (6, ':description', 'dsDescriptionValue', FALSE, NULL, 1 ); -INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) VALUES (7, ':relation', 'relatedMaterial', FALSE, NULL, 1 ); -INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) VALUES (8, ':isReferencedBy', 'publicationCitation', FALSE, NULL, 1 ); -INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) VALUES (9, 'holdingsURI', 'publicationURL', TRUE, 8, 1 ); -INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) VALUES (10, 'agency', 'publicationIDType', TRUE, 8, 1 ); -INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) VALUES (11, 'IDNo', 'publicationIDNumber', TRUE, 8, 1 ); -INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) VALUES (12, ':coverage', 'otherGeographicCoverage', FALSE, NULL, 1 ); -INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) VALUES (13, ':type', 'kindOfData', FALSE, NULL, 1 ); -INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) VALUES (14, ':source', 'dataSources', FALSE, NULL, 1 ); -INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) VALUES (15, 'affiliation', 'authorAffiliation', TRUE, 3, 1 ); -INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) VALUES (16, ':contributor', 'contributorName', FALSE, NULL, 1 ); -INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) VALUES (17, 'type', 'contributorType', TRUE, 16, 1 ); -INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) VALUES (18, ':publisher', 'producerName', FALSE, NULL, 1 ); -INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) VALUES (19, ':language', 'language', FALSE, NULL, 1 ); - -INSERT INTO guestbook( - emailrequired, enabled, institutionrequired, createtime, - "name", namerequired, positionrequired, dataverse_id) - VALUES ( false, true, false, now(), - 'Default', false, false, null); - -INSERT INTO worldmapauth_tokentype -( name, - created, - contactemail, hostname, ipaddress, - mapitlink, md5, - modified, timelimitminutes) - VALUES ( 'GEOCONNECT', current_timestamp, - 'support@dataverse.org', 'geoconnect.datascience.iq.harvard.edu', '140.247.115.127', - 'http://geoconnect.datascience.iq.harvard.edu/shapefile/map-it', - '38c0a931b2d582a5c43fc79405b30c22', - current_timestamp, 30); diff --git a/src/main/resources/db/migration/afterMigrate__1-upsert-referenceData.sql b/src/main/resources/db/migration/afterMigrate__1-upsert-referenceData.sql new file mode 100644 index 00000000000..60140b44485 --- /dev/null +++ b/src/main/resources/db/migration/afterMigrate__1-upsert-referenceData.sql @@ -0,0 +1,42 @@ +-- using http://dublincore.org/schemas/xmls/qdc/dcterms.xsd because at http://dublincore.org/schemas/xmls/ it's the +-- schema location for http://purl.org/dc/terms/ which is referenced in http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html +INSERT INTO foreignmetadataformatmapping (id, name, startelement, displayName, schemalocation) + VALUES + (1, 'http://purl.org/dc/terms/', 'entry', 'dcterms: DCMI Metadata Terms', 'http://dublincore.org/schemas/xmls/qdc/dcterms.xsd') + ON CONFLICT DO NOTHING; + +INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id) + VALUES + (1, ':title', 'title', FALSE, NULL, 1 ), + (2, ':identifier', 'otherIdValue', FALSE, NULL, 1 ), + (3, ':creator', 'authorName', FALSE, NULL, 1 ), + (4, ':date', 'productionDate', FALSE, NULL, 1 ), + (5, ':subject', 'keywordValue', FALSE, NULL, 1 ), + (6, ':description', 'dsDescriptionValue', FALSE, NULL, 1 ), + (7, ':relation', 'relatedMaterial', FALSE, NULL, 1 ), + (8, ':isReferencedBy', 'publicationCitation', FALSE, NULL, 1 ), + (9, 'holdingsURI', 'publicationURL', TRUE, 8, 1 ), + (10, 'agency', 'publicationIDType', TRUE, 8, 1 ), + (11, 'IDNo', 'publicationIDNumber', TRUE, 8, 1 ), + (12, ':coverage', 'otherGeographicCoverage', FALSE, NULL, 1 ), + (13, ':type', 'kindOfData', FALSE, NULL, 1 ), + (14, ':source', 'dataSources', FALSE, NULL, 1 ), + (15, 'affiliation', 'authorAffiliation', TRUE, 3, 1 ), + (16, ':contributor', 'contributorName', FALSE, NULL, 1 ), + (17, 'type', 'contributorType', TRUE, 16, 1 ), + (18, ':publisher', 'producerName', FALSE, NULL, 1 ), + (19, ':language', 'language', FALSE, NULL, 1 ) + ON CONFLICT DO NOTHING; + +INSERT INTO guestbook (emailrequired, enabled, institutionrequired, createtime, name, namerequired, positionrequired, dataverse_id) + SELECT false, true, false, now(), 'Default', false, false, null + WHERE NOT EXISTS (SELECT id FROM guestbook); + +INSERT INTO worldmapauth_tokentype + (name, created, contactemail, hostname, + ipaddress, mapitlink, + md5, modified, timelimitminutes) + SELECT 'GEOCONNECT', current_timestamp, 'support@dataverse.org', 'geoconnect.datascience.iq.harvard.edu', + '140.247.115.127', 'http://geoconnect.datascience.iq.harvard.edu/shapefile/map-it', + '38c0a931b2d582a5c43fc79405b30c22', current_timestamp, 30 + WHERE NOT EXISTS (SELECT id from worldmapauth_tokentype); From 92e960934f739328c02e8845358ca79f18a99bbc Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 6 Nov 2020 16:40:39 +0100 Subject: [PATCH 4/6] Move index creation from baseline script to a 5.1.1 migration, so it will be applied in order (cannot create an older, out-of-order version as this would break migrations for everyone). #7256 --- .../db/migration/V1__flyway_schema_baseline.sql | 9 --------- .../V5.1.1.3__7256-purge-referencedata.sql | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 9 deletions(-) create mode 100644 src/main/resources/db/migration/V5.1.1.3__7256-purge-referencedata.sql diff --git a/src/main/resources/db/migration/V1__flyway_schema_baseline.sql b/src/main/resources/db/migration/V1__flyway_schema_baseline.sql index 2ec219cd19e..d1caa63cd71 100644 --- a/src/main/resources/db/migration/V1__flyway_schema_baseline.sql +++ b/src/main/resources/db/migration/V1__flyway_schema_baseline.sql @@ -1,11 +1,2 @@ -- TODO: we still should add the real base line here, too. That would avoid conflicts between EclipseLink -- trying to create new tables on existing databases. See https://github.com/IQSS/dataverse/issues/5871 - --- This is unsupported by JPA, as it is PostgreSQL specific. Has to be done here, cannot be done in code. --- (Only other option would be a lowercase copy of the data as a separate column, automatically filled py JPA) -CREATE UNIQUE INDEX IF NOT EXISTS dataverse_alias_unique_idx on dataverse (LOWER(alias)); -CREATE UNIQUE INDEX IF NOT EXISTS index_authenticateduser_lower_email ON authenticateduser (lower(email)); - --- Edit Dataset: Investigate and correct multiple draft issue: https://github.com/IQSS/dataverse/issues/2132 --- This unique index will prevent the multiple draft issue, yet it cannot be done in JPA code. -CREATE UNIQUE INDEX IF NOT EXISTS one_draft_version_per_dataset ON datasetversion (dataset_id) WHERE versionstate='DRAFT'; \ No newline at end of file diff --git a/src/main/resources/db/migration/V5.1.1.3__7256-purge-referencedata.sql b/src/main/resources/db/migration/V5.1.1.3__7256-purge-referencedata.sql new file mode 100644 index 00000000000..176629f9abb --- /dev/null +++ b/src/main/resources/db/migration/V5.1.1.3__7256-purge-referencedata.sql @@ -0,0 +1,17 @@ +-- #5361 and #7256 is about faster deployments, especially during development, sitting on an empty database. +-- +-- This script has been part of scripts/database/reference_data.sql that had to be executed manually on every new +-- deployment (manually in the sense of Flyway didn't, the outside installer or an admin took care of it). +-- +-- While this is pretty old stuff and should have been done earlier (baseline...), it will be a nice migration +-- and behave like nothing happened if this is an existing installation. All new installation have an empty database +-- on first app boot and benefit from this Flyway-based management. + +-- This is unsupported by JPA, as it is PostgreSQL specific. Has to be done here, cannot be done in code. +-- (Only other option would be a lowercase copy of the data as a separate column, automatically filled py JPA) +CREATE UNIQUE INDEX IF NOT EXISTS dataverse_alias_unique_idx on dataverse (LOWER(alias)); +CREATE UNIQUE INDEX IF NOT EXISTS index_authenticateduser_lower_email ON authenticateduser (lower(email)); + +-- Edit Dataset: Investigate and correct multiple draft issue: https://github.com/IQSS/dataverse/issues/2132 +-- This unique index will prevent the multiple draft issue, yet it cannot be done in JPA code. +CREATE UNIQUE INDEX IF NOT EXISTS one_draft_version_per_dataset ON datasetversion (dataset_id) WHERE versionstate='DRAFT'; \ No newline at end of file From 5132e04e6e61f5bb26b153b9e8b743429445286f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 6 Nov 2020 16:44:45 +0100 Subject: [PATCH 5/6] Add a comment to the first bootstrap SQL script. #7256 --- .../migration/afterMigrate__1-upsert-referenceData.sql | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/resources/db/migration/afterMigrate__1-upsert-referenceData.sql b/src/main/resources/db/migration/afterMigrate__1-upsert-referenceData.sql index 60140b44485..d45c03bc03b 100644 --- a/src/main/resources/db/migration/afterMigrate__1-upsert-referenceData.sql +++ b/src/main/resources/db/migration/afterMigrate__1-upsert-referenceData.sql @@ -1,3 +1,10 @@ +-- #5361 and #7256 is about faster deployments, especially during development, sitting on an empty database. +-- +-- This script has been part of scripts/database/reference_data.sql that had to be executed manually on every new +-- deployment (manually in the sense of Flyway didn't, the outside installer or an admin took care of it). +-- +-- This script will load some initial, common data if not present (so only once, when booting for the first time). + -- using http://dublincore.org/schemas/xmls/qdc/dcterms.xsd because at http://dublincore.org/schemas/xmls/ it's the -- schema location for http://purl.org/dc/terms/ which is referenced in http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html INSERT INTO foreignmetadataformatmapping (id, name, startelement, displayName, schemalocation) @@ -28,10 +35,12 @@ INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname (19, ':language', 'language', FALSE, NULL, 1 ) ON CONFLICT DO NOTHING; +-- Simple trick: WHERE NOT EXISTS (SELECT id FROM table) is only true if the table is empty. INSERT INTO guestbook (emailrequired, enabled, institutionrequired, createtime, name, namerequired, positionrequired, dataverse_id) SELECT false, true, false, now(), 'Default', false, false, null WHERE NOT EXISTS (SELECT id FROM guestbook); +-- Simple trick: WHERE NOT EXISTS (SELECT id FROM table) is only true if the table is empty. INSERT INTO worldmapauth_tokentype (name, created, contactemail, hostname, ipaddress, mapitlink, From 422bc26046aa8d0bfba31a870ed3ef5f647b9da3 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Sat, 14 Nov 2020 10:59:21 +0100 Subject: [PATCH 6/6] Update Flyway SQL files to reflect version change to 5.2. #7256 --- ...e-referencedata.sql => V5.2.0.1__7256-purge-referencedata.sql} | 0 ...enceData.sql => afterMigrate__1-7256-upsert-referenceData.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.1.1.3__7256-purge-referencedata.sql => V5.2.0.1__7256-purge-referencedata.sql} (100%) rename src/main/resources/db/migration/{afterMigrate__1-upsert-referenceData.sql => afterMigrate__1-7256-upsert-referenceData.sql} (100%) diff --git a/src/main/resources/db/migration/V5.1.1.3__7256-purge-referencedata.sql b/src/main/resources/db/migration/V5.2.0.1__7256-purge-referencedata.sql similarity index 100% rename from src/main/resources/db/migration/V5.1.1.3__7256-purge-referencedata.sql rename to src/main/resources/db/migration/V5.2.0.1__7256-purge-referencedata.sql diff --git a/src/main/resources/db/migration/afterMigrate__1-upsert-referenceData.sql b/src/main/resources/db/migration/afterMigrate__1-7256-upsert-referenceData.sql similarity index 100% rename from src/main/resources/db/migration/afterMigrate__1-upsert-referenceData.sql rename to src/main/resources/db/migration/afterMigrate__1-7256-upsert-referenceData.sql