From d757099c7aefd080f47fcf13f14a8406b42b3e76 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 26 Oct 2017 12:08:27 -0400 Subject: [PATCH 1/5] Glassfish 5 stub, don't assume postgres user #4217 #3905 --- conf/docker/build.sh | 6 +++--- conf/docker/dataverse-glassfish/.gitignore | 2 +- conf/docker/dataverse-glassfish/Dockerfile | 14 +++++--------- .../source/_static/util/default.config | 5 +++-- .../source/developers/dev-environment.rst | 2 ++ .../source/installation/prerequisites.rst | 2 ++ downloads/.gitignore | 2 +- downloads/download.sh | 2 +- scripts/installer/.gitignore | 1 + scripts/installer/install | 15 +++++++++++++-- scripts/installer/resetgf5anddb | 6 ++++++ scripts/vagrant/setup.sh | 2 ++ 12 files changed, 40 insertions(+), 19 deletions(-) create mode 100644 scripts/installer/.gitignore create mode 100755 scripts/installer/resetgf5anddb diff --git a/conf/docker/build.sh b/conf/docker/build.sh index a4828ba607f..f0c44081957 100755 --- a/conf/docker/build.sh +++ b/conf/docker/build.sh @@ -28,14 +28,14 @@ else fi echo Images will be pushed to Docker Hub with the tag \"$TAG\". # Use "conf" directory as context so we can copy schema.xml into Solr image. -docker build -t iqss/dataverse-solr:$TAG -f solr/Dockerfile ../../conf -docker push iqss/dataverse-solr:$TAG +#docker build -t iqss/dataverse-solr:$TAG -f solr/Dockerfile ../../conf +#docker push iqss/dataverse-solr:$TAG # TODO: Think about if we really need dataverse.war because it's in dvinstall.zip. # FIXME: Automate the building of dataverse.war and dvinstall.zip. Think about https://github.com/IQSS/dataverse/issues/3974 and https://github.com/IQSS/dataverse/pull/3975 cp ../../target/dataverse*.war dataverse-glassfish/dataverse.war cp ../../scripts/installer/dvinstall.zip dataverse-glassfish cp ../../doc/sphinx-guides/source/_static/util/default.config dataverse-glassfish -cp ../../downloads/glassfish-4.1.zip dataverse-glassfish +cp ../../downloads/glassfish-5.0.zip dataverse-glassfish cp ../../downloads/weld-osgi-bundle-2.2.10.Final-glassfish4.jar dataverse-glassfish docker build -t iqss/dataverse-glassfish:$TAG dataverse-glassfish # FIXME: Check the output of `docker build` and only push on success. diff --git a/conf/docker/dataverse-glassfish/.gitignore b/conf/docker/dataverse-glassfish/.gitignore index b0e6e38894f..16e404425d5 100644 --- a/conf/docker/dataverse-glassfish/.gitignore +++ b/conf/docker/dataverse-glassfish/.gitignore @@ -1,4 +1,4 @@ -glassfish-4.1.zip +glassfish-5.0.zip weld-osgi-bundle-2.2.10.Final-glassfish4.jar dvinstall.zip dataverse.war diff --git a/conf/docker/dataverse-glassfish/Dockerfile b/conf/docker/dataverse-glassfish/Dockerfile index 939ce98fb72..01eee622aee 100644 --- a/conf/docker/dataverse-glassfish/Dockerfile +++ b/conf/docker/dataverse-glassfish/Dockerfile @@ -1,7 +1,7 @@ FROM centos:7.2.1511 MAINTAINER Dataverse (support@dataverse.org) -COPY glassfish-4.1.zip /tmp +COPY glassfish-5.0.zip /tmp COPY weld-osgi-bundle-2.2.10.Final-glassfish4.jar /tmp COPY default.config /tmp COPY dvinstall.zip /tmp @@ -33,16 +33,12 @@ RUN find /tmp # #RUN exitEarly -# Install Glassfish 4.1 +# Install Glassfish 5.0 RUN cd /tmp \ - && unzip glassfish-4.1.zip \ - && mv glassfish4 /usr/local \ - && cd /usr/local/glassfish4/glassfish/modules \ - && rm weld-osgi-bundle.jar \ - && cp /tmp/weld-osgi-bundle-2.2.10.Final-glassfish4.jar . \ - #FIXME: Patch Grizzly too! - && echo "Done installing and patching Glassfish" + && unzip glassfish-5.0.zip \ + && mv glassfish5 /usr/local \ + && echo "Done installing Glassfish" RUN chmod g=u /etc/passwd diff --git a/doc/sphinx-guides/source/_static/util/default.config b/doc/sphinx-guides/source/_static/util/default.config index 8f95e1dd3a3..f9c1f809381 100644 --- a/doc/sphinx-guides/source/_static/util/default.config +++ b/doc/sphinx-guides/source/_static/util/default.config @@ -1,7 +1,8 @@ HOST_DNS_ADDRESS localhost -GLASSFISH_DIRECTORY /usr/local/glassfish4 +GLASSFISH_DIRECTORY /usr/local/glassfish5 ADMIN_EMAIL -MAIL_SERVER mail.hmdc.harvard.edu +MAIL_SERVER localhost +POSTGRES_ADMIN_USERNAME postgres POSTGRES_ADMIN_PASSWORD secret POSTGRES_SERVER 127.0.0.1 POSTGRES_PORT 5432 diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 73cf0925549..12c571dcf48 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -27,6 +27,8 @@ Glassfish As a `Java Enterprise Edition `_ 7 (Java EE 7) application, Dataverse requires an applications server to run. +FIXME: Update to Glassfish 5 + Glassfish 4.1 is required (not any earlier or later versions until https://github.com/IQSS/dataverse/issues/2628 is resolved), which can be downloaded from http://download.oracle.com/glassfish/4.1/release/glassfish-4.1.zip . If you have downloaded Glassfish as part of a Netbeans bundle, you can manually add the proper version by clicking "Tools", "Servers", "Add Server". By default, Glassfish reports analytics information. The administration guide suggests this can be disabled with ``asadmin create-jvm-options -Dcom.sun.enterprise.tools.admingui.NO_NETWORK=true``, should this be found to be undesirable for development purposes. diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index db3573bff8b..a16d109b780 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -45,6 +45,8 @@ Installing Glassfish **Note:** The Dataverse installer need not be run as root, and it is recommended that Glassfish not run as root either. We suggest the creation of a glassfish service account for this purpose. +FIXME: Update to Glassfish 5 + - Download and install Glassfish (installed in ``/usr/local/glassfish4`` in the example commands below):: # wget http://dlc-cdn.sun.com/glassfish/4.1/release/glassfish-4.1.zip diff --git a/downloads/.gitignore b/downloads/.gitignore index 21c1afb26d9..fef1eba4bf7 100644 --- a/downloads/.gitignore +++ b/downloads/.gitignore @@ -1,4 +1,4 @@ -glassfish-4.1.zip +glassfish-5.0.zip solr-4.6.0.tgz weld-osgi-bundle-2.2.10.Final-glassfish4.jar schemaSpy_5.0.0.jar diff --git a/downloads/download.sh b/downloads/download.sh index 176ac5f6dea..ec1562ebb53 100755 --- a/downloads/download.sh +++ b/downloads/download.sh @@ -1,5 +1,5 @@ #!/bin/sh -curl -L -O http://download.java.net/glassfish/4.1/release/glassfish-4.1.zip +curl -L -O http://download.oracle.com/glassfish/5.0/release/glassfish-5.0.zip curl -L -O https://archive.apache.org/dist/lucene/solr/4.6.0/solr-4.6.0.tgz curl -L -O http://search.maven.org/remotecontent?filepath=org/jboss/weld/weld-osgi-bundle/2.2.10.Final/weld-osgi-bundle-2.2.10.Final-glassfish4.jar curl -s -L http://sourceforge.net/projects/schemaspy/files/schemaspy/SchemaSpy%205.0.0/schemaSpy_5.0.0.jar/download > schemaSpy_5.0.0.jar diff --git a/scripts/installer/.gitignore b/scripts/installer/.gitignore new file mode 100644 index 00000000000..9853e3bc8fd --- /dev/null +++ b/scripts/installer/.gitignore @@ -0,0 +1 @@ +default.config diff --git a/scripts/installer/install b/scripts/installer/install index f86883ec17f..c8decc6769c 100755 --- a/scripts/installer/install +++ b/scripts/installer/install @@ -54,6 +54,7 @@ else 'POSTGRES_SERVER', 'POSTGRES_PORT', + 'POSTGRES_ADMIN_USERNAME', 'POSTGRES_ADMIN_PASSWORD', 'POSTGRES_DATABASE', 'POSTGRES_USER', @@ -79,6 +80,7 @@ my %CONFIG_DEFAULTS = ( 'ADMIN_EMAIL', '', 'MAIL_SERVER', 'mail.hmdc.harvard.edu', + 'POSTGRES_ADMIN_USERNAME', 'postgres', 'POSTGRES_ADMIN_PASSWORD', 'secret', 'POSTGRES_SERVER', '127.0.0.1', 'POSTGRES_PORT', 5432, @@ -105,6 +107,7 @@ my %CONFIG_PROMPTS = ( 'POSTGRES_SERVER', 'Postgres Server Address', 'POSTGRES_PORT', 'Postgres Server Port', + 'POSTGRES_ADMIN_USERNAME', 'Postgres ADMIN username', 'POSTGRES_ADMIN_PASSWORD', 'Postgres ADMIN password', 'POSTGRES_DATABASE', 'Name of the Postgres Database', 'POSTGRES_USER', 'Name of the Postgres User', @@ -131,6 +134,7 @@ my %CONFIG_COMMENTS = ( 'POSTGRES_SERVER', '', 'POSTGRES_PORT', '', + 'POSTGRES_ADMIN_USERNAME', ":\n - We will need this to create the user and database that the Dataverse application will be using.\n (Hit RETURN if access control is set to \"trust\" for this connection in pg_hba.conf)\n: ", 'POSTGRES_ADMIN_PASSWORD', ":\n - We will need this to create the user and database that the Dataverse application will be using.\n (Hit RETURN if access control is set to \"trust\" for this connection in pg_hba.conf)\n: ", 'POSTGRES_USER', ":\n - This is the Postgres user that the Dataverse app will be using to talk to the database\n: ", 'POSTGRES_DATABASE', '', @@ -713,14 +717,20 @@ if ( $psql_exec eq "" ) $psql_admin_exec = "PGPASSWORD=" . $CONFIG_DEFAULTS{'POSTGRES_ADMIN_PASSWORD'} . "; export PGPASSWORD; " . $psql_exec; $psql_exec = "PGPASSWORD=" . $CONFIG_DEFAULTS{'POSTGRES_PASSWORD'} . "; export PGPASSWORD; " . $psql_exec; - print "Checking if we can talk to Postgres as the admin user...\n"; + print "Checking if we can talk to Postgres as the admin user (" . $CONFIG_DEFAULTS{'POSTGRES_ADMIN_USERNAME'} . ")...\n"; } # 4. CONFIGURE POSTGRES: # 4a. BUT FIRST, CHECK IF WE CAN TALK TO POSTGRES AS THE ADMIN: -if ( $psql_admin_exec eq "" || system( $psql_admin_exec . "/psql -h " . $CONFIG_DEFAULTS{'POSTGRES_SERVER'} . " -U postgres -d postgres -c 'SELECT * FROM pg_roles' > /dev/null 2>&1" ) ) +print "\$psql_admin_exec: $psql_admin_exec\n"; + +my $foo = $psql_admin_exec . "/psql -h " . $CONFIG_DEFAULTS{'POSTGRES_SERVER'} . " -U " . $CONFIG_DEFAULTS{'POSTGRES_ADMIN_USERNAME'} . " -d postgres -c 'SELECT * FROM pg_roles'"; + +print "\$foo: $foo\n"; + +if ( $psql_admin_exec eq "" || system( $psql_admin_exec . "/psql -h " . $CONFIG_DEFAULTS{'POSTGRES_SERVER'} . " -U " . $CONFIG_DEFAULTS{'POSTGRES_ADMIN_USERNAME'} . " -d postgres -c 'SELECT * FROM pg_roles' > /dev/null 2>&1" ) ) { # No, we can't. :( if ($pg_local_connection) @@ -1087,6 +1097,7 @@ my $DOMAIN_DOWN = print STDERR $DOMAIN_DOWN . "\n"; if ($DOMAIN_DOWN) { print "Trying to start domain up...\n"; + # FIXME: After this point the script is no longer non-interactive. You have to hit enter when you see "Enter admin user name [Enter to accept default]>". system( "sudo -u $CONFIG_DEFAULTS{'GLASSFISH_USER'} " . $CONFIG_DEFAULTS{'GLASSFISH_DIRECTORY'} . "/bin/asadmin start-domain domain1" ); # TODO: (?) - retest that the domain is running now? diff --git a/scripts/installer/resetgf5anddb b/scripts/installer/resetgf5anddb new file mode 100755 index 00000000000..d78b8e06778 --- /dev/null +++ b/scripts/installer/resetgf5anddb @@ -0,0 +1,6 @@ +#!/bin/sh +/usr/local/glassfish5/glassfish/bin/asadmin stop-domain +rm -rf /usr/local/glassfish5/glassfish/domains/domain1/generated +psql -c 'DROP DATABASE "dvndb"' template1 +../search/clear +rm -rf ~/dataverse/files diff --git a/scripts/vagrant/setup.sh b/scripts/vagrant/setup.sh index 5789450aa51..6deebd5827a 100644 --- a/scripts/vagrant/setup.sh +++ b/scripts/vagrant/setup.sh @@ -28,6 +28,7 @@ GLASSFISH_USER=glassfish echo "Ensuring Unix user '$GLASSFISH_USER' exists" useradd $GLASSFISH_USER || : DOWNLOAD_DIR='/dataverse/downloads' +# FIXME: update to Glassfish 5 GLASSFISH_ZIP="$DOWNLOAD_DIR/glassfish-4.1.zip" SOLR_TGZ="$DOWNLOAD_DIR/solr-4.6.0.tgz" WELD_PATCH="$DOWNLOAD_DIR/weld-osgi-bundle-2.2.10.Final-glassfish4.jar" @@ -37,6 +38,7 @@ if [ ! -f $GLASSFISH_ZIP ] || [ ! -f $SOLR_TGZ ]; then echo "Done running download script." fi GLASSFISH_USER_HOME=~glassfish +# FIXME: update to Glassfish 5 GLASSFISH_ROOT=$GLASSFISH_USER_HOME/glassfish4 if [ ! -d $GLASSFISH_ROOT ]; then echo "Copying $GLASSFISH_ZIP to $GLASSFISH_USER_HOME and unzipping" From 5ad645ddae882c3c04777f86d5471f881b51c6da Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 27 Oct 2017 12:13:03 -0400 Subject: [PATCH 2/5] avoid interactive "Enter admin user name [Enter to accept default]>" #3799 --- scripts/installer/install | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/installer/install b/scripts/installer/install index c8decc6769c..1d8f3c0e25d 100755 --- a/scripts/installer/install +++ b/scripts/installer/install @@ -1097,7 +1097,6 @@ my $DOMAIN_DOWN = print STDERR $DOMAIN_DOWN . "\n"; if ($DOMAIN_DOWN) { print "Trying to start domain up...\n"; - # FIXME: After this point the script is no longer non-interactive. You have to hit enter when you see "Enter admin user name [Enter to accept default]>". system( "sudo -u $CONFIG_DEFAULTS{'GLASSFISH_USER'} " . $CONFIG_DEFAULTS{'GLASSFISH_DIRECTORY'} . "/bin/asadmin start-domain domain1" ); # TODO: (?) - retest that the domain is running now? @@ -1110,7 +1109,11 @@ else { # the username and password for every asadmin command, if # access to :4848 is password-protected: -system( $glassfish_dir. "/bin/asadmin login" ); +# If you you have a "default.config" file in place and run `./install -y -f -nogfpasswd` +# you can keep the installation process non-interactive. +unless ($nogfpasswd) { + system( $glassfish_dir. "/bin/asadmin login" ); +} # NEW: configure glassfish using ASADMIN commands: From f7649090d0f8d87a158ec11d5e75be9ef4a3aea1 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 27 Oct 2017 13:45:30 -0400 Subject: [PATCH 3/5] allow insecure install for dev environments #2443 --- .../source/developers/dev-environment.rst | 2 +- scripts/installer/install | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 12c571dcf48..35d5d9d86e4 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -165,7 +165,7 @@ Run Installer Please note the following: - If you have trouble with the SMTP server, consider editing the installer script to disable the SMTP check. -- Rather than running the installer in "interactive" mode, it's possible to put the values in a file. See "non-interactive mode" in the :doc:`/installation/installation-main` section of the Installation Guide. +- Rather than running the installer in "interactive" mode, it's possible to put the values in a file. See "non-interactive mode" in the :doc:`/installation/installation-main` section of the Installation Guide. If you put a ``default.config`` file in place you can run ``./install -y -f -nogfpasswd -insecure`` to set up a dev environment, for example. Now that you have all the prerequisites in place, you need to configure the environment for the Dataverse app - configure the database connection, set some options, etc. We have an installer script that should do it all for you. Again, assuming that the clone on the Dataverse repository was retrieved using NetBeans and that it is saved in the path ~/NetBeansProjects: diff --git a/scripts/installer/install b/scripts/installer/install index 1d8f3c0e25d..b786fa9e6e1 100755 --- a/scripts/installer/install +++ b/scripts/installer/install @@ -15,6 +15,7 @@ my $mailserver; my $yes; my $force; my $nogfpasswd; +my $insecure; my $admin_email; my ($rez) = GetOptions( #"length=i" => \$length, # numeric @@ -28,6 +29,7 @@ my ($rez) = GetOptions( "y|yes" => \$yes, "f|force" => \$force, "nogfpasswd" => \$nogfpasswd, + "insecure" => \$insecure, "admin_email=s" => \$admin_email, ); @@ -1299,10 +1301,15 @@ for my $script ( "setup-all.sh" ) { # $run_script = "tmpscript.sh"; #} #else { - $run_script = $script; + # $run_script = $script; #} + if ($insecure) { + $run_script = "$script --insecure"; + } else { + $run_script = $script; + } - unless ( my $exit_code = system( "./" . $run_script . " > $run_script.$$.log 2>&1") == 0 ) + unless ( my $exit_code = system( "./" . $run_script . " > $script.$$.log 2>&1") == 0 ) { print "\nERROR executing script " . $script . "!\n"; exit 1; From 6f72baa611f7304e7e5795467d5c9b0ef013ac2e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 27 Oct 2017 14:29:35 -0400 Subject: [PATCH 4/5] JSF is setting `page` to "0" on Glassfish 5. Why? #4217 --- .../harvard/iq/dataverse/search/SearchIncludeFragment.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index c4f48a8cd63..0313e92cb2c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -307,6 +307,10 @@ public void search(boolean onlyDataRelatedToMe) { String allTypesFilterQuery = SearchFields.TYPE + ":(dataverses OR datasets OR files)"; filterQueriesFinalAllTypes.add(allTypesFilterQuery); + if (page == 0) { + logger.info("Variable 'page' was zero. Changing it one. What does this have to do with upgrading to Glassfish 5?"); + page = 1; + } int paginationStart = (page - 1) * paginationGuiRows; /** * @todo From 153027390a701e0a54ce71f3249173de2a5993a5 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 27 Oct 2017 14:59:56 -0400 Subject: [PATCH 5/5] get rid of "Validation Error" on homepage #4217 --- src/main/webapp/dataverse.xhtml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index 55818bdd09c..3b747d91d39 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -18,7 +18,11 @@ - + +