IQSS · kcondon · Nov 16, 2020 · Oct 23, 2020 · Oct 23, 2020 · Oct 23, 2020
diff --git a/scripts/database/reference_data.sql b/scripts/database/reference_data.sql
diff --git a/scripts/installer/Makefile b/scripts/installer/Makefile
@@ -3,7 +3,6 @@ DISTRIBUTION_WAR_FILE=${INSTALLER_ZIP_DIR}/dataverse.war
 GLASSFISH_SETUP_SCRIPT=${INSTALLER_ZIP_DIR}/as-setup.sh
 POSTGRES_DRIVERS=${INSTALLER_ZIP_DIR}/pgdriver
 API_SCRIPTS=${INSTALLER_ZIP_DIR}/setup-datasetfields.sh ${INSTALLER_ZIP_DIR}/setup-users.sh ${INSTALLER_ZIP_DIR}/setup-builtin-roles.sh ${INSTALLER_ZIP_DIR}/setup-dvs.sh ${INSTALLER_ZIP_DIR}/data ${INSTALLER_ZIP_DIR}/setup-identity-providers.sh  ${INSTALLER_ZIP_DIR}/setup-all.sh ${INSTALLER_ZIP_DIR}/post-install-api-block.sh
-DB_SCRIPT=${INSTALLER_ZIP_DIR}/reference_data.sql
 JHOVE_CONFIG=${INSTALLER_ZIP_DIR}/jhove.conf
 JHOVE_SCHEMA=${INSTALLER_ZIP_DIR}/jhoveConfig.xsd
 SOLR_SCHEMA=${INSTALLER_ZIP_DIR}/schema.xml ${INSTALLER_ZIP_DIR}/schema_dv_mdb_fields.xml ${INSTALLER_ZIP_DIR}/schema_dv_mdb_copies.xml ${INSTALLER_ZIP_DIR}/updateSchemaMDB.sh
@@ -16,9 +15,9 @@ installer:	dvinstall.zip
 clean: 
 	/bin/rm -rf ${INSTALLER_ZIP_DIR} dvinstall.zip
 
-dvinstall.zip:	${GLASSFISH_SETUP_SCRIPT} ${POSTGRES_DRIVERS} ${JSF_PATCH_DIR} ${DISTRIBUTION_WAR_FILE} ${API_SCRIPTS} ${DB_SCRIPT} ${JHOVE_CONFIG} ${JHOVE_SCHEMA} ${SOLR_SCHEMA} ${SOLR_CONFIG} ${PYTHON_FILES} ${INSTALL_SCRIPT}
+dvinstall.zip:	${GLASSFISH_SETUP_SCRIPT} ${POSTGRES_DRIVERS} ${JSF_PATCH_DIR} ${DISTRIBUTION_WAR_FILE} ${API_SCRIPTS} ${JHOVE_CONFIG} ${JHOVE_SCHEMA} ${SOLR_SCHEMA} ${SOLR_CONFIG} ${PYTHON_FILES} ${INSTALL_SCRIPT}
 	@echo making installer...
-	zip -r dvinstall.zip ${GLASSFISH_SETUP_SCRIPT} ${POSTGRES_DRIVERS} ${JSF_PATCH_DIR} ${DISTRIBUTION_WAR_FILE} ${API_SCRIPTS} ${DB_SCRIPT} ${JHOVE_CONFIG} ${JHOVE_SCHEMA} ${SOLR_SCHEMA} ${SOLR_CONFIG} ${PYTHON_FILES} ${INSTALL_SCRIPT}
+	zip -r dvinstall.zip ${GLASSFISH_SETUP_SCRIPT} ${POSTGRES_DRIVERS} ${JSF_PATCH_DIR} ${DISTRIBUTION_WAR_FILE} ${API_SCRIPTS} ${JHOVE_CONFIG} ${JHOVE_SCHEMA} ${SOLR_SCHEMA} ${SOLR_CONFIG} ${PYTHON_FILES} ${INSTALL_SCRIPT}
 	@echo 
 	@echo "Done!"
 
@@ -55,10 +54,6 @@ ${API_SCRIPTS}:	../api/setup-datasetfields.sh ../api/setup-users.sh ../api/setup
 	@echo copying api scripts
 	/bin/cp -R ../api/setup-datasetfields.sh ../api/setup-users.sh ../api/setup-dvs.sh ../api/setup-identity-providers.sh ../api/setup-all.sh ../api/post-install-api-block.sh ../api/setup-builtin-roles.sh ../api/data ${INSTALLER_ZIP_DIR}
 
-${DB_SCRIPT}:	../database/reference_data.sql ${INSTALLER_ZIP_DIR}
-	@echo copying reference data sql script
-	/bin/cp ../database/reference_data.sql ${INSTALLER_ZIP_DIR}
-
 ${JHOVE_CONFIG}:	../../conf/jhove/jhove.conf ${INSTALLER_ZIP_DIR}
 	@echo copying jhove config file
 	/bin/cp ../../conf/jhove/jhove.conf ${INSTALLER_ZIP_DIR}

diff --git a/scripts/installer/README.txt b/scripts/installer/README.txt
@@ -32,10 +32,6 @@ setup-identity-providers.sh
 setup-users.sh
 data (the entire directory with all its contents)
 
-from scripts/database:
-
-reference_data.sql
-
 from conf/jhove:
 
 jhove.conf

diff --git a/scripts/installer/install b/scripts/installer/install
@@ -46,8 +46,7 @@ if (exists($ENV{'MY_POD_NAME'}))
 my $jq_exec_path = "";
 my $psql_exec_path = "";
 my $cwd;
-my $WARFILE_LOCATION = "dataverse.war"; 
-my $REFERENCE_DATA_SQL = "../database/reference_data.sql";
+my $WARFILE_LOCATION = "dataverse.war";
 
 
 my @CONFIG_VARIABLES;
@@ -317,23 +316,6 @@ unless ($postgresonly)
 # 1b. check and remember the working dir:
     chomp( $cwd = `pwd` );
 
-# 1c. check if the reference data SQL file is there:
-
-    $REFERENCE_DATA_SQL = "../database/reference_data.sql";
-
-    unless ( -f $REFERENCE_DATA_SQL ) {
-	# if it's not in ../database, then we're probably running out of the 
-	# unzipped installer bundle, so it should be right here in the current directory:
-	$REFERENCE_DATA_SQL = "reference_data.sql";
-    }
-
-    unless ( -f $REFERENCE_DATA_SQL ) {
-	print "\nWARNING: Can't find .sql data template!\n";
-	print "(are you running the installer in the right directory?)\n";
-
-	exit 0;
-    }
-
 # 1d. jq executable: 
 
     my $sys_path = $ENV{'PATH'};
@@ -563,15 +545,6 @@ my $glassfish_dir = $CONFIG_DEFAULTS{'GLASSFISH_DIRECTORY'};
 
 my $done = &setup_appserver();
 
-# 7. POPULATE THE DATABASE WITH REFERENCE DATA
-# TODO: move this further down
-# (in this step some pre-supplied content is inserted into the database that we have just created;
-# it is not *necessary* for the application to run in the very basic mode; but some features - certain 
-# types of metadata imports, for example - will be unavailable if it's not done. 
-
-&import_reference_data();
-
-
 # Check if the App is running: 
 
 unless ((
@@ -1513,48 +1486,6 @@ sub setup_postgres {
     }
 }
 
-sub import_reference_data {
-    print "\npopulating the database with reference data:\n\n";
-
-    # (we have already verified that the referenceData.sql file exists)
-
-    my $psql_command = $psql_exec_path . "/psql -h " . $CONFIG_DEFAULTS{'POSTGRES_SERVER'}
-        . " -p " . $CONFIG_DEFAULTS{'POSTGRES_PORT'}
-        . " -U " . $CONFIG_DEFAULTS{'POSTGRES_USER'}
-        . " -d " . $CONFIG_DEFAULTS{'POSTGRES_DATABASE'} . " -f " . $REFERENCE_DATA_SQL;
-
-    print "EXECUTING PSQL COMMAND: $psql_command\n";
-    unless ( ( my $exitcode = system("$psql_command") ) == 0 )
-    {
-        print "WARNING: Could not pre-populate Postgres database for the Dataverse application!\n";
-        print "(command: " . $psql_command . ")\n";
-        print "(psql exit code: " . $exitcode . ")\n";
-        print "\nYou must populate the database in order for all the features of your \n";
-        print "new Dataverse to be available. \n";
-        print "\n";
-        print "You can try this again, by executing the following on the command line:\n";
-        print "   psql -U $CONFIG_DEFAULTS{'POSTGRES_USER'} -d $CONFIG_DEFAULTS{'POSTGRES_DATABASE'} -f $REFERENCE_DATA_SQL\n";
-        print "then re-start payara  \n\n";
-        print "\n";
-        print "If it's still failing, please consult the installation manual and/or\n";
-        print "seek support from the Dataverse team.\n\n";
-
-        print "Press any key to continue... ";
-
-	unless ($noninteractive) 
-	{
-	    system "stty cbreak </dev/tty >/dev/tty 2>&1";
-	    my $key = getc(STDIN);
-	    system "stty -cbreak </dev/tty >/dev/tty 2>&1";
-	    print "\n";
-	}
-    }
-    else
-    {
-        print "\nOK, done!\n";
-    }
-}
-
 sub read_config_defaults {
     my $config_file = shift @_;
 

diff --git a/scripts/installer/install.py b/scripts/installer/install.py
@@ -245,17 +245,6 @@
          sys.exit("Sorry, I can't seem to find an appropriate warfile.\nAre you running the installer from the right directory?")
    print(warfile+" available to deploy. Good.")
 
-   # 1b. check for reference_data.sql
-   referenceData = '../database/reference_data.sql'
-   if not os.path.isfile(referenceData):
-      # if it's not there, then we're probably running out of the 
-      # unzipped installer bundle, so it should be right here in the current directory:
-      referenceData = 'reference_data.sql'
-      if not os.path.isfile(referenceData):
-         sys.exit("Can't find reference_data.sql!\nAre you running the installer from the right directory?")
-
-   print("found "+referenceData+"... good")
-
    # 1c. check if jq is available
    # (but we're only doing it if it's not that weird "pod name" mode)
    if podName != "start-glassfish":
@@ -558,22 +547,6 @@
 # @todo: restart/try to deploy again if it failed?
 # @todo: if asadmin deploy says it was successful, verify that the application is running... if not - repeat the above?
 
-# 6. Import reference data
-print("importing reference data...")
-# open the new postgresQL connection (as the application user):
-conn_string="dbname='"+pgDb+"' user='"+pgUser+"' password='"+pgPassword+"' host='"+pgHost+"'"
-conn = psycopg2.connect(conn_string)
-conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
-cur = conn.cursor()
-try:
-   cur.execute(open(referenceData, "r").read())
-   print("done.")
-except: 
-   print("WARNING: failed to import reference data!")
-
-cur.close()
-conn.close()
-
 # 7. RUN SETUP SCRIPTS AND CONFIGURE EXTRA SETTINGS
 # (note that we may need to change directories, depending on whether this is a dev., or release installer)
 # 7a. run setup scripts

diff --git a/src/main/resources/db/migration/V1__flyway_schema_baseline.sql b/src/main/resources/db/migration/V1__flyway_schema_baseline.sql
@@ -0,0 +1,2 @@
+-- TODO: we still should add the real base line here, too. That would avoid conflicts between EclipseLink
+--       trying to create new tables on existing databases. See https://github.com/IQSS/dataverse/issues/5871
diff --git a/src/main/resources/db/migration/V5.1.1.3__7256-purge-referencedata.sql b/src/main/resources/db/migration/V5.1.1.3__7256-purge-referencedata.sql
@@ -0,0 +1,17 @@
+-- #5361 and #7256 is about faster deployments, especially during development, sitting on an empty database.
+--
+-- This script has been part of  scripts/database/reference_data.sql that had to be executed manually on every new
+-- deployment (manually in the sense of Flyway didn't, the outside installer or an admin took care of it).
+--
+-- While this is pretty old stuff and should have been done earlier (baseline...), it will be a nice migration
+-- and behave like nothing happened if this is an existing installation. All new installation have an empty database
+-- on first app boot and benefit from this Flyway-based management.
+
+-- This is unsupported by JPA, as it is PostgreSQL specific. Has to be done here, cannot be done in code.
+-- (Only other option would be a lowercase copy of the data as a separate column, automatically filled py JPA)
+CREATE UNIQUE INDEX IF NOT EXISTS dataverse_alias_unique_idx on dataverse (LOWER(alias));
+CREATE UNIQUE INDEX IF NOT EXISTS index_authenticateduser_lower_email ON authenticateduser (lower(email));
+
+-- Edit Dataset: Investigate and correct multiple draft issue: https://github.com/IQSS/dataverse/issues/2132
+-- This unique index will prevent the multiple draft issue, yet it cannot be done in JPA code.
+CREATE UNIQUE INDEX IF NOT EXISTS one_draft_version_per_dataset ON datasetversion (dataset_id) WHERE versionstate='DRAFT';
diff --git a/src/main/resources/db/migration/afterMigrate__1-upsert-referenceData.sql b/src/main/resources/db/migration/afterMigrate__1-upsert-referenceData.sql
@@ -0,0 +1,51 @@
+-- #5361 and #7256 is about faster deployments, especially during development, sitting on an empty database.
+--
+-- This script has been part of  scripts/database/reference_data.sql that had to be executed manually on every new
+-- deployment (manually in the sense of Flyway didn't, the outside installer or an admin took care of it).
+--
+-- This script will load some initial, common data if not present (so only once, when booting for the first time).
+
+-- using http://dublincore.org/schemas/xmls/qdc/dcterms.xsd because at http://dublincore.org/schemas/xmls/ it's the
+-- schema location for http://purl.org/dc/terms/ which is referenced in http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html
+INSERT INTO foreignmetadataformatmapping (id, name, startelement, displayName, schemalocation)
+       VALUES
+              (1, 'http://purl.org/dc/terms/', 'entry', 'dcterms: DCMI Metadata Terms', 'http://dublincore.org/schemas/xmls/qdc/dcterms.xsd')
+       ON CONFLICT DO NOTHING;
+
+INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname, isattribute, parentfieldmapping_id, foreignmetadataformatmapping_id)
+       VALUES
+              (1, ':title', 'title', FALSE, NULL, 1 ),
+              (2, ':identifier', 'otherIdValue', FALSE, NULL, 1 ),
+              (3, ':creator', 'authorName', FALSE, NULL, 1 ),
+              (4, ':date', 'productionDate', FALSE, NULL, 1 ),
+              (5, ':subject', 'keywordValue', FALSE, NULL, 1 ),
+              (6, ':description', 'dsDescriptionValue', FALSE, NULL, 1 ),
+              (7, ':relation', 'relatedMaterial', FALSE, NULL, 1 ),
+              (8, ':isReferencedBy', 'publicationCitation', FALSE, NULL, 1 ),
+              (9, 'holdingsURI', 'publicationURL', TRUE, 8, 1 ),
+              (10, 'agency', 'publicationIDType', TRUE, 8, 1 ),
+              (11, 'IDNo', 'publicationIDNumber', TRUE, 8, 1 ),
+              (12, ':coverage', 'otherGeographicCoverage', FALSE, NULL, 1 ),
+              (13, ':type', 'kindOfData', FALSE, NULL, 1 ),
+              (14, ':source', 'dataSources', FALSE, NULL, 1 ),
+              (15, 'affiliation', 'authorAffiliation', TRUE, 3, 1 ),
+              (16, ':contributor', 'contributorName', FALSE, NULL, 1 ),
+              (17, 'type', 'contributorType', TRUE, 16, 1 ),
+              (18, ':publisher', 'producerName', FALSE, NULL, 1 ),
+              (19, ':language', 'language', FALSE, NULL, 1 )
+       ON CONFLICT DO NOTHING;
+
+-- Simple trick: WHERE NOT EXISTS (SELECT id FROM table) is only true if the table is empty.
+INSERT INTO guestbook (emailrequired, enabled, institutionrequired, createtime, name, namerequired, positionrequired, dataverse_id)
+       SELECT false, true, false, now(), 'Default', false, false, null
+       WHERE NOT EXISTS (SELECT id FROM guestbook);
+
+-- Simple trick: WHERE NOT EXISTS (SELECT id FROM table) is only true if the table is empty.
+INSERT INTO worldmapauth_tokentype
+              (name, created, contactemail, hostname,
+               ipaddress, mapitlink,
+               md5, modified, timelimitminutes)
+       SELECT 'GEOCONNECT', current_timestamp, 'support@dataverse.org',  'geoconnect.datascience.iq.harvard.edu',
+               '140.247.115.127', 'http://geoconnect.datascience.iq.harvard.edu/shapefile/map-it',
+               '38c0a931b2d582a5c43fc79405b30c22', current_timestamp, 30
+       WHERE NOT EXISTS (SELECT id from worldmapauth_tokentype);
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		-- TODO: we still should add the real base line here, too. That would avoid conflicts between EclipseLink
		-- trying to create new tables on existing databases. See https://github.com/IQSS/dataverse/issues/5871