diff --git a/.env b/.env index e3ececc2e54..ae266af80da 100644 --- a/.env +++ b/.env @@ -2,3 +2,4 @@ APP_IMAGE=gdcc/dataverse:unstable POSTGRES_VERSION=13 DATAVERSE_DB_USER=dataverse SOLR_VERSION=9.3.0 +SKIP_DEPLOY=0 \ No newline at end of file diff --git a/conf/proxy/Caddyfile b/conf/proxy/Caddyfile new file mode 100644 index 00000000000..70e6904d26e --- /dev/null +++ b/conf/proxy/Caddyfile @@ -0,0 +1,12 @@ +# This configuration is intended to be used with Caddy, a very small high perf proxy. +# It will serve the application containers Payara Admin GUI via HTTP instead of HTTPS, +# avoiding the trouble of self signed certificates for local development. + +:4848 { + reverse_proxy https://dataverse:4848 { + transport http { + tls_insecure_skip_verify + } + header_down Location "^https://" "http://" + } +} diff --git a/doc/release-notes/9590-faster-redeploy.md b/doc/release-notes/9590-faster-redeploy.md new file mode 100644 index 00000000000..ed903849444 --- /dev/null +++ b/doc/release-notes/9590-faster-redeploy.md @@ -0,0 +1,5 @@ +In the Container Guide, documentation for developers on how to quickly redeploy code has been added for Netbeans and improved for IntelliJ. + +Also in the context of containers, a new option to skip deployment has been added and the war file is now consistently named "dataverse.war" rather than having a version in the filename, such as "dataverse-6.1.war". This predictability makes tooling easier. + +Finally, an option to create tabs in the guides using [Sphinx Tabs](https://sphinx-tabs.readthedocs.io) has been added. (You can see the tabs in action in the "dev usage" page of the Container Guide.) To continue building the guides, you will need to install this new dependency by re-running `pip install -r requirements.txt`. \ No newline at end of file diff --git a/doc/sphinx-guides/requirements.txt b/doc/sphinx-guides/requirements.txt index f55d2b9e518..20daa189037 100755 --- a/doc/sphinx-guides/requirements.txt +++ b/doc/sphinx-guides/requirements.txt @@ -5,3 +5,6 @@ sphinx-icon==0.1.2 # Markdown support myst-parser==2.0.0 + +# tabs +sphinx-tabs==3.4.5 diff --git a/doc/sphinx-guides/source/_static/docsdataverse_org.css b/doc/sphinx-guides/source/_static/docsdataverse_org.css index da4ba06ddd4..726abcc42bd 100755 --- a/doc/sphinx-guides/source/_static/docsdataverse_org.css +++ b/doc/sphinx-guides/source/_static/docsdataverse_org.css @@ -182,3 +182,11 @@ div.form-group .glyphicon.glyphicon-asterisk {font-size: .5em; vertical-align: t pre { white-space: pre-wrap; } + +div.sphinx-tabs { + width: 100%; +} + +li div.sphinx-tabs { + padding-left: 0; +} \ No newline at end of file diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index 78eadd9b2ce..66911aa0ad1 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -413,7 +413,7 @@ Setting Up a Dev Environment for Testing You have several options for setting up a dev environment for testing metadata block changes: -- Docker: See :doc:`/container/index`. +- Docker: See :doc:`/container/running/metadata-blocks` in the Container Guide. - AWS deployment: See the :doc:`/developers/deployment` section of the Developer Guide. - Full dev environment: See the :doc:`/developers/dev-environment` section of the Developer Guide. diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py index 67d7a722074..98d10526517 100755 --- a/doc/sphinx-guides/source/conf.py +++ b/doc/sphinx-guides/source/conf.py @@ -43,6 +43,7 @@ 'sphinx.ext.graphviz', 'sphinxcontrib.icon', 'myst_parser', + 'sphinx_tabs.tabs', ] # Add any paths that contain templates here, relative to this directory. diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index 1a47a8fc413..c41250d48c5 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -217,7 +217,14 @@ provides. These are mostly based on environment variables (very common with cont - ``0`` - Bool, ``0|1`` - Enable the dynamic "hot" reloads of files when changed in a deployment. Useful for development, - when new artifacts are copied into the running domain. + when new artifacts are copied into the running domain. Also, export Dataverse specific environment variables + ``DATAVERSE_JSF_PROJECT_STAGE=Development`` and ``DATAVERSE_JSF_REFRESH_PERIOD=0`` to enable dynamic JSF page + reloads. + * - ``SKIP_DEPLOY`` + - ``0`` + - Bool, ``0|1`` or ``false|true`` + - When active, do not deploy applications from ``DEPLOY_DIR`` (see below), just start the application server. + Will still execute any provided init scripts and only skip deployments within the default init scripts. * - ``DATAVERSE_HTTP_TIMEOUT`` - ``900`` - Seconds @@ -272,7 +279,8 @@ building upon it. You can also use these for references in scripts, etc. (Might be reused for Dataverse one day) * - ``DEPLOY_DIR`` - ``${HOME_DIR}/deployments`` - - Any EAR or WAR file, exploded WAR directory etc are autodeployed on start + - Any EAR or WAR file, exploded WAR directory etc are autodeployed on start. + See also ``SKIP_DEPLOY`` above. * - ``DOMAIN_DIR`` - ``${PAYARA_DIR}/glassfish`` ``/domains/${DOMAIN_NAME}`` - Path to root of the Payara domain applications will be deployed into. Usually ``${DOMAIN_NAME}`` will be ``domain1``. @@ -299,9 +307,9 @@ named Docker volume in these places to avoid data loss, gain performance and/or - Description * - ``STORAGE_DIR`` - ``/dv`` - - This place is writeable by the Payara user, making it usable as a place to store research data, customizations - or other. Images inheriting the base image should create distinct folders here, backed by different - mounted volumes. + - This place is writeable by the Payara user, making it usable as a place to store research data, customizations or other. + Images inheriting the base image should create distinct folders here, backed by different mounted volumes. + Enforce correct filesystem permissions on the mounted volume using ``fix-fs-perms.sh`` from :doc:`configbaker-image` or similar scripts. * - ``SECRETS_DIR`` - ``/secrets`` - Mount secrets or other here, being picked up automatically by @@ -353,6 +361,8 @@ Other Hints By default, ``domain1`` is enabled to use the ``G1GC`` garbage collector. +To access the Payara Admin Console or use the ``asadmin`` command, use username ``admin`` and password ``admin``. + For running a Java application within a Linux based container, the support for CGroups is essential. It has been included and activated by default since Java 8u192, Java 11 LTS and later. If you are interested in more details, you can read about those in a few places like https://developers.redhat.com/articles/2022/04/19/java-17-whats-new-openjdks-container-awareness, diff --git a/doc/sphinx-guides/source/container/dev-usage.rst b/doc/sphinx-guides/source/container/dev-usage.rst index b2547306b03..be4eda5da44 100644 --- a/doc/sphinx-guides/source/container/dev-usage.rst +++ b/doc/sphinx-guides/source/container/dev-usage.rst @@ -144,43 +144,202 @@ Alternatives: Redeploying ----------- -Rebuild and Running Images -^^^^^^^^^^^^^^^^^^^^^^^^^^ +The safest and most reliable way to redeploy code is to stop the running containers (with Ctrl-c if you started them in the foreground) and then build and run them again with ``mvn -Pct clean package docker:run``. +Safe, but also slowing down the development cycle a lot. -The safest way to redeploy code is to stop the running containers (with Ctrl-c if you started them in the foreground) and then build and run them again with ``mvn -Pct clean package docker:run``. +Triggering redeployment of changes using an IDE can greatly improve your feedback loop when changing code. +You have at least two options: -IntelliJ IDEA Ultimate and Payara Platform Tools -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +#. Use builtin features of IDEs or `IDE plugins from Payara `_. +#. Use a paid product like `JRebel `_. -If you have IntelliJ IDEA Ultimate (note that `free educational licenses `_ are available), you can install `Payara Platform Tools `_ which can dramatically improve your feedback loop when iterating on code. +The main differences between the first and the second options are support for hot deploys of non-class files and limitations in what the JVM HotswapAgent can do for you. +Find more details in a `blog article by JRebel `_. -The following steps are suggested: +.. _ide-trigger-code-deploy: -- Go to the Payara admin console (either at https://localhost:4848 or http://localhost:4849) and undeploy the dataverse application under "Applications". -- Install Payara Platform Tools. -- Under "Server": +IDE Triggered Code Re-Deployments +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - Click "Run" then "Edit Configurations". - - Click the plus sign and scroll down to Payara Server and click "Remote". - - For "Name" put "Payara in Docker" or something reasonable. - - Under "Application server" select a local directory that has the same version of Payara used in the container. This should match the version of Payara mentioned in the Installation Guide under :ref:`payara`. - - Change "Admin Server Port" to 4849. - - For username, put "admin". - - For password, put "admin". +To make use of builtin features or Payara IDE Tools (option 1), please follow steps below. +Note that using this method, you may redeploy a complete WAR or single methods. +Redeploying WARs supports swapping and adding classes and non-code materials, but is slower (still faster than rebuilding containers). +Hotswapping methods requires using JDWP (Debug Mode), but does not allow switching non-code material or adding classes. -- Under "Deployment": +#. | Download the version of Payara shown in :ref:`install-payara-dev` and unzip it to a reasonable location such as ``/usr/local/payara6``. + | - Note that Payara can also be downloaded from `Maven Central `_. + | - Note that another way to check the expected version of Payara is to run this command: + | ``mvn help:evaluate -Dexpression=payara.version -q -DforceStdout`` - - Click the plus button and clien "Artifact" then "dataverse:war". +#. Install Payara Tools plugin in your IDE: -- Under "Startup/Connection": + .. tabs:: + .. group-tab:: Netbeans - - Click "Debug" and change the port to 9009. + This step is not necessary for Netbeans. The feature is builtin. -- Click "Run" and then "Debug Payara in Docker". This initial deployment will take some time. -- Go to http://localhost:8080/api/info/version and make sure the API is responding. -- Edit ``Info.java`` and make a small change to the ``/api/info/version`` code. -- Click "Run" then "Debugging Actions" then "Reload Changed Classes". The deployment should only take a few seconds. -- Go to http://localhost:8080/api/info/version and verify the change you made. + .. group-tab:: IntelliJ + + **Requires IntelliJ Ultimate!** + (Note that `free educational licenses `_ are available) + + .. image:: img/intellij-payara-plugin-install.png + +#. Configure a connection to Payara: + + .. tabs:: + .. group-tab:: Netbeans + + Launch Netbeans and click "Tools" and then "Servers". Click "Add Server" and select "Payara Server" and set the installation location to ``/usr/local/payara6`` (or wherever you unzipped Payara). Choose "Remote Domain". Use the settings in the screenshot below. Most of the defaults are fine. + + Under "Common", the username and password should be "admin". Make sure "Enable Hot Deploy" is checked. + + .. image:: img/netbeans-servers-common.png + + Under "Java", change the debug port to 9009. + + .. image:: img/netbeans-servers-java.png + + Open the project properties (under "File"), navigate to "Compile" and make sure "Compile on Save" is checked. + + .. image:: img/netbeans-compile.png + + Under "Run", under "Server", select "Payara Server". Make sure "Deploy on Save" is checked. + + .. image:: img/netbeans-run.png + + .. group-tab:: IntelliJ + Create a new running configuration with a "Remote Payara". + (Open dialog by clicking "Run", then "Edit Configurations") + + .. image:: img/intellij-payara-add-new-config.png + + Click on "Configure" next to "Application Server". + Add an application server and select unzipped local directory. + + .. image:: img/intellij-payara-config-add-server.png + + Add admin password "admin" and add "building artifact" before launch. + Make sure to select the WAR, *not* exploded! + + .. image:: img/intellij-payara-config-server.png + + Go to "Deployment" tab and add the Dataverse WAR, *not* exploded!. + + .. image:: img/intellij-payara-config-deployment.png + + Go to "Startup/Connection" tab, select "Debug" and change port to ``9009``. + + .. image:: img/intellij-payara-config-startup.png + + You might want to tweak the hot deploy behavior in the "Server" tab now. + "Update action" can be found in the run window (see below). + "Frame deactivation" means switching from IntelliJ window to something else, e.g. your browser. + *Note: static resources like properties, XHTML etc will only update when redeploying!* + + .. image:: img/intellij-payara-config-server-behaviour.png + +#. Start all the containers, but take care to skip application deployment. + + .. tabs:: + .. group-tab:: Maven + ``mvn -Pct docker:run -Dapp.skipDeploy`` + + Run above command in your terminal to start containers in foreground and skip deployment. + See cheat sheet above for more options. + Note that this command either assumes you built the :doc:`app-image` first or will download it from Docker Hub. + .. group-tab:: Compose + ``SKIP_DEPLOY=1 docker compose -f docker-compose-dev.yml up`` + + Run above command in your terminal to start containers in foreground and skip deployment. + See cheat sheet above for more options. + Note that this command either assumes you built the :doc:`app-image` first or will download it from Docker Hub. + .. group-tab:: IntelliJ + You can create a service configuration to automatically start services for you. + + **IMPORTANT**: This requires installation of the `Docker plugin `_. + + **NOTE**: You might need to change the Docker Compose executable in your IDE settings to ``docker`` if you have no ``docker-compose`` bin (*File > Settings > Build > Docker > Tools*). + + .. image:: img/intellij-compose-add-new-config.png + + Give your configuration a meaningful name, select the compose file to use (in this case the default one), add the environment variable ``SKIP_DEPLOY=1``, and optionally select the services to start. + You might also want to change other options like attaching to containers to view the logs within the "Services" tab. + + .. image:: img/intellij-compose-setup.png + + Now run the configuration to prepare for deployment and watch it unfold in the "Services" tab. + + .. image:: img/intellij-compose-run.png + .. image:: img/intellij-compose-services.png + + Note: the Admin Console can be reached at http://localhost:4848 or https://localhost:4949 + +#. To deploy the application to the running server, use the configured tools to deploy. + Using the "Run" configuration only deploys and enables redeploys, while running "Debug" enables hot swapping of classes via JDWP. + + .. tabs:: + .. group-tab:: Netbeans + + Click "Debug" then "Debug Project". After some time, Dataverse will be deployed. + + Try making a code change, perhaps to ``Info.java``. + + Click "Debug" and then "Apply Code Changes". If the change was correctly applied, you should see output similar to this: + + .. code-block:: + + Classes to reload: + edu.harvard.iq.dataverse.api.Info + + Code updated + + Check to make sure the change is live by visiting, for example, http://localhost:8080/api/info/version + + See below for a `video `_ demonstrating the steps above but please note that the ports used have changed and now that we have the concept of "skip deploy" the undeployment step shown is no longer necessary. + + .. raw:: html + + + + .. group-tab:: IntelliJ + Choose "Run" or "Debug" in the toolbar. + + .. image:: img/intellij-payara-run-toolbar.png + + Watch the WAR build and the deployment unfold. + Note the "Update" action button (see config to change its behavior). + + .. image:: img/intellij-payara-run-output.png + + Manually hotswap classes in "Debug" mode via "Run" > "Debugging Actions" > "Reload Changed Classes". + + .. image:: img/intellij-payara-run-menu-reload.png + +Note: in the background, the bootstrap job will wait for Dataverse to be deployed and responsive. +When your IDE automatically opens the URL a newly deployed, not bootstrapped Dataverse application, it might take some more time and page refreshes until the job finishes. + +IDE Triggered Non-Code Re-Deployments +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Either redeploy the WAR (see above), use JRebel or look into copying files into the exploded WAR within the running container. +The steps below describe options to enable the later in different IDEs. + +.. tabs:: + .. group-tab:: IntelliJ + + This imitates the Netbeans builtin function to copy changes to files under ``src/main/webapp`` into a destination folder. + It is different in the way that it will copy the files into the running container deployment without using a bind mount. + + 1. Install the `File Watchers plugin `_ + 2. Import the :download:`watchers.xml <../../../../docker/util/intellij/watchers.xml>` file at *File > Settings > Tools > File Watchers* + 3. Once you have the deployment running (see above), editing files under ``src/main/webapp`` will be copied into the container after saving the edited file. + Note: by default, IDE auto-saves will not trigger the copy. + 4. Changes are visible once you reload the browser window. + + **IMPORTANT**: This tool assumes you are using the :ref:`ide-trigger-code-deploy` method to run Dataverse. + + **IMPORTANT**: This tool uses a Bash shell script and is thus limited to Mac and Linux OS. Using a Debugger ---------------- diff --git a/doc/sphinx-guides/source/container/img/intellij-compose-add-new-config.png b/doc/sphinx-guides/source/container/img/intellij-compose-add-new-config.png new file mode 100644 index 00000000000..cec9bb357fe Binary files /dev/null and b/doc/sphinx-guides/source/container/img/intellij-compose-add-new-config.png differ diff --git a/doc/sphinx-guides/source/container/img/intellij-compose-run.png b/doc/sphinx-guides/source/container/img/intellij-compose-run.png new file mode 100644 index 00000000000..e01744134f9 Binary files /dev/null and b/doc/sphinx-guides/source/container/img/intellij-compose-run.png differ diff --git a/doc/sphinx-guides/source/container/img/intellij-compose-services.png b/doc/sphinx-guides/source/container/img/intellij-compose-services.png new file mode 100644 index 00000000000..1c500c54201 Binary files /dev/null and b/doc/sphinx-guides/source/container/img/intellij-compose-services.png differ diff --git a/doc/sphinx-guides/source/container/img/intellij-compose-setup.png b/doc/sphinx-guides/source/container/img/intellij-compose-setup.png new file mode 100644 index 00000000000..42c2accf2b4 Binary files /dev/null and b/doc/sphinx-guides/source/container/img/intellij-compose-setup.png differ diff --git a/doc/sphinx-guides/source/container/img/intellij-payara-add-new-config.png b/doc/sphinx-guides/source/container/img/intellij-payara-add-new-config.png new file mode 100644 index 00000000000..d1c7a8f2777 Binary files /dev/null and b/doc/sphinx-guides/source/container/img/intellij-payara-add-new-config.png differ diff --git a/doc/sphinx-guides/source/container/img/intellij-payara-config-add-server.png b/doc/sphinx-guides/source/container/img/intellij-payara-config-add-server.png new file mode 100644 index 00000000000..54ffbd1b713 Binary files /dev/null and b/doc/sphinx-guides/source/container/img/intellij-payara-config-add-server.png differ diff --git a/doc/sphinx-guides/source/container/img/intellij-payara-config-deployment.png b/doc/sphinx-guides/source/container/img/intellij-payara-config-deployment.png new file mode 100644 index 00000000000..52adee056b5 Binary files /dev/null and b/doc/sphinx-guides/source/container/img/intellij-payara-config-deployment.png differ diff --git a/doc/sphinx-guides/source/container/img/intellij-payara-config-server-behaviour.png b/doc/sphinx-guides/source/container/img/intellij-payara-config-server-behaviour.png new file mode 100644 index 00000000000..5d23672e614 Binary files /dev/null and b/doc/sphinx-guides/source/container/img/intellij-payara-config-server-behaviour.png differ diff --git a/doc/sphinx-guides/source/container/img/intellij-payara-config-server.png b/doc/sphinx-guides/source/container/img/intellij-payara-config-server.png new file mode 100644 index 00000000000..614bda6f6d7 Binary files /dev/null and b/doc/sphinx-guides/source/container/img/intellij-payara-config-server.png differ diff --git a/doc/sphinx-guides/source/container/img/intellij-payara-config-startup.png b/doc/sphinx-guides/source/container/img/intellij-payara-config-startup.png new file mode 100644 index 00000000000..35b87148859 Binary files /dev/null and b/doc/sphinx-guides/source/container/img/intellij-payara-config-startup.png differ diff --git a/doc/sphinx-guides/source/container/img/intellij-payara-plugin-install.png b/doc/sphinx-guides/source/container/img/intellij-payara-plugin-install.png new file mode 100644 index 00000000000..7c6896574de Binary files /dev/null and b/doc/sphinx-guides/source/container/img/intellij-payara-plugin-install.png differ diff --git a/doc/sphinx-guides/source/container/img/intellij-payara-run-menu-reload.png b/doc/sphinx-guides/source/container/img/intellij-payara-run-menu-reload.png new file mode 100644 index 00000000000..b1fd8bea260 Binary files /dev/null and b/doc/sphinx-guides/source/container/img/intellij-payara-run-menu-reload.png differ diff --git a/doc/sphinx-guides/source/container/img/intellij-payara-run-output.png b/doc/sphinx-guides/source/container/img/intellij-payara-run-output.png new file mode 100644 index 00000000000..aa139485a9d Binary files /dev/null and b/doc/sphinx-guides/source/container/img/intellij-payara-run-output.png differ diff --git a/doc/sphinx-guides/source/container/img/intellij-payara-run-toolbar.png b/doc/sphinx-guides/source/container/img/intellij-payara-run-toolbar.png new file mode 100644 index 00000000000..2aecb27c5f3 Binary files /dev/null and b/doc/sphinx-guides/source/container/img/intellij-payara-run-toolbar.png differ diff --git a/doc/sphinx-guides/source/container/img/netbeans-compile.png b/doc/sphinx-guides/source/container/img/netbeans-compile.png new file mode 100644 index 00000000000..e429695ccb0 Binary files /dev/null and b/doc/sphinx-guides/source/container/img/netbeans-compile.png differ diff --git a/doc/sphinx-guides/source/container/img/netbeans-run.png b/doc/sphinx-guides/source/container/img/netbeans-run.png new file mode 100644 index 00000000000..00f8af23cc5 Binary files /dev/null and b/doc/sphinx-guides/source/container/img/netbeans-run.png differ diff --git a/doc/sphinx-guides/source/container/img/netbeans-servers-common.png b/doc/sphinx-guides/source/container/img/netbeans-servers-common.png new file mode 100644 index 00000000000..a9ded5dbec3 Binary files /dev/null and b/doc/sphinx-guides/source/container/img/netbeans-servers-common.png differ diff --git a/doc/sphinx-guides/source/container/img/netbeans-servers-java.png b/doc/sphinx-guides/source/container/img/netbeans-servers-java.png new file mode 100644 index 00000000000..2593cacc5ae Binary files /dev/null and b/doc/sphinx-guides/source/container/img/netbeans-servers-java.png differ diff --git a/doc/sphinx-guides/source/developers/classic-dev-env.rst b/doc/sphinx-guides/source/developers/classic-dev-env.rst index d7b7f281634..6978f389e01 100755 --- a/doc/sphinx-guides/source/developers/classic-dev-env.rst +++ b/doc/sphinx-guides/source/developers/classic-dev-env.rst @@ -88,6 +88,8 @@ On Mac, run this command: On Linux, install ``jq`` from your package manager or download a binary from https://stedolan.github.io/jq/ +.. _install-payara-dev: + Install Payara ~~~~~~~~~~~~~~ @@ -260,7 +262,3 @@ Next Steps If you can log in to the Dataverse installation, great! If not, please see the :doc:`troubleshooting` section. For further assistance, please see "Getting Help" in the :doc:`intro` section. You're almost ready to start hacking on code. Now that the installer script has you up and running, you need to continue on to the :doc:`tips` section to get set up to deploy code from your IDE or the command line. - ----- - -Previous: :doc:`intro` | Next: :doc:`tips` diff --git a/doc/sphinx-guides/source/developers/coding-style.rst b/doc/sphinx-guides/source/developers/coding-style.rst index 0c00f611a7f..9da7836bbf4 100755 --- a/doc/sphinx-guides/source/developers/coding-style.rst +++ b/doc/sphinx-guides/source/developers/coding-style.rst @@ -57,7 +57,7 @@ Place curly braces according to the style below, which is an example you can see Format Code You Changed with Netbeans ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -As you probably gathered from the :doc:`dev-environment` section, IQSS has standardized on Netbeans. It is much appreciated when you format your code (but only the code you touched!) using the out-of-the-box Netbeans configuration. If you have created an entirely new Java class, you can just click Source -> Format. If you are adjusting code in an existing class, highlight the code you changed and then click Source -> Format. Keeping the "diff" in your pull requests small makes them easier to code review. +IQSS has standardized on Netbeans. It is much appreciated when you format your code (but only the code you touched!) using the out-of-the-box Netbeans configuration. If you have created an entirely new Java class, you can just click Source -> Format. If you are adjusting code in an existing class, highlight the code you changed and then click Source -> Format. Keeping the "diff" in your pull requests small makes them easier to code review. Checking Your Formatting With Checkstyle ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -131,7 +131,3 @@ Bike Shedding What color should the `bike shed `_ be? :) Come debate with us about coding style in this Google doc that has public comments enabled: https://docs.google.com/document/d/1KTd3FpM1BI3HlBofaZjMmBiQEJtFf11jiiGpQeJzy7A/edit?usp=sharing - ----- - -Previous: :doc:`debugging` | Next: :doc:`deployment` diff --git a/doc/sphinx-guides/source/developers/containers.rst b/doc/sphinx-guides/source/developers/containers.rst index 175b178b455..ed477ccefea 100755 --- a/doc/sphinx-guides/source/developers/containers.rst +++ b/doc/sphinx-guides/source/developers/containers.rst @@ -29,7 +29,3 @@ Using Containers for Reproducible Research ------------------------------------------ Please see :ref:`research-code` in the User Guide for this related topic. - ----- - -Previous: :doc:`deployment` | Next: :doc:`making-releases` diff --git a/doc/sphinx-guides/source/developers/debugging.rst b/doc/sphinx-guides/source/developers/debugging.rst index 50e8901b1ff..ffee6764b7f 100644 --- a/doc/sphinx-guides/source/developers/debugging.rst +++ b/doc/sphinx-guides/source/developers/debugging.rst @@ -63,7 +63,3 @@ to maintain your settings more easily for different environments. .. _Jakarta Server Faces 3.0 Spec: https://jakarta.ee/specifications/faces/3.0/jakarta-faces-3.0.html#a6088 .. _PrimeFaces Configuration Docs: https://primefaces.github.io/primefaces/11_0_0/#/gettingstarted/configuration - ----- - -Previous: :doc:`documentation` | Next: :doc:`coding-style` diff --git a/doc/sphinx-guides/source/developers/dependencies.rst b/doc/sphinx-guides/source/developers/dependencies.rst index 0208c49f90a..26880374f23 100644 --- a/doc/sphinx-guides/source/developers/dependencies.rst +++ b/doc/sphinx-guides/source/developers/dependencies.rst @@ -444,7 +444,3 @@ The codebase is structured like this: .. [#f1] Modern IDEs import your Maven POM and offer import autocompletion for classes based on direct dependencies in the model. You might end up using legacy or repackaged classes because of a wrong scope. .. [#f2] This is going to bite back in modern IDEs when importing classes from transitive dependencies by "autocompletion accident". - ----- - -Previous: :doc:`documentation` | Next: :doc:`debugging` diff --git a/doc/sphinx-guides/source/developers/deployment.rst b/doc/sphinx-guides/source/developers/deployment.rst index 045b0d0abbc..75ad2fa33fb 100755 --- a/doc/sphinx-guides/source/developers/deployment.rst +++ b/doc/sphinx-guides/source/developers/deployment.rst @@ -146,8 +146,3 @@ To Update Datafile Location to your-s3-bucket, Assuming no ``file://`` Prefix WHERE id IN (SELECT o.id FROM dvobject o, dataset s WHERE o.dtype = 'DataFile' AND s.id = o.owner_id AND s.harvestingclient_id IS null AND o.storageidentifier NOT LIKE '%://%'); - - ----- - -Previous: :doc:`coding-style` | Next: :doc:`containers` diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 1301994cc82..2837f901d5e 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -71,10 +71,10 @@ After some time you should be able to log in: - username: dataverseAdmin - password: admin1 -More Information ----------------- +Next Steps +---------- -See also the :doc:`/container/dev-usage` section of the Container Guide. +See the :doc:`/container/dev-usage` section of the Container Guide for tips on fast redeployment, viewing logs, and more. Getting Help ------------ diff --git a/doc/sphinx-guides/source/developers/documentation.rst b/doc/sphinx-guides/source/developers/documentation.rst index 4ec011f2b24..a4b8c027445 100755 --- a/doc/sphinx-guides/source/developers/documentation.rst +++ b/doc/sphinx-guides/source/developers/documentation.rst @@ -159,7 +159,3 @@ A few notes about the command above: Also, as of this writing we have enabled PDF builds from the "develop" branch. You download the PDF from http://preview.guides.gdcc.io/_/downloads/en/develop/pdf/ If you would like to help improve the PDF version of the guides, please get in touch! Please see :ref:`getting-help-developers` for ways to contact the developer community. - ----- - -Previous: :doc:`testing` | Next: :doc:`dependencies` diff --git a/doc/sphinx-guides/source/developers/fontcustom.rst b/doc/sphinx-guides/source/developers/fontcustom.rst index 2a94b0ffc0b..edcda1e69ab 100755 --- a/doc/sphinx-guides/source/developers/fontcustom.rst +++ b/doc/sphinx-guides/source/developers/fontcustom.rst @@ -35,7 +35,7 @@ RVM is a good way to install a specific version of Ruby: https://rvm.io Install Dependencies and Font Custom Gem ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The brew commands below assume you are on a Mac. See :doc:`dev-environment` for more on ``brew``. +The brew commands below assume you are on a Mac. .. code-block:: bash diff --git a/doc/sphinx-guides/source/developers/geospatial.rst b/doc/sphinx-guides/source/developers/geospatial.rst index 9744438bf5d..0570e993d10 100644 --- a/doc/sphinx-guides/source/developers/geospatial.rst +++ b/doc/sphinx-guides/source/developers/geospatial.rst @@ -81,7 +81,3 @@ For two "final" shapefile sets, ``bicycles.zip`` and ``subway_line.zip``, a new - Mimetype: ``application/zipped-shapefile`` - Mimetype Label: "Shapefile as ZIP Archive" - ----- - -Previous: :doc:`unf/index` | Next: :doc:`remote-users` diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst index ce05670977a..350968012d8 100755 --- a/doc/sphinx-guides/source/developers/intro.rst +++ b/doc/sphinx-guides/source/developers/intro.rst @@ -75,7 +75,3 @@ As a developer, you also may be interested in these projects related to Datavers - Third party apps - make use of Dataverse installation APIs: :doc:`/api/apps` - chat.dataverse.org - chat interface for Dataverse Project users and developers: https://github.com/IQSS/chat.dataverse.org - [Your project here] :) - ----- - -Next: :doc:`dev-environment` diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index 18ae34ee656..e7a59910e56 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -230,7 +230,3 @@ We've merged the hotfix into the "master" branch but now we need the fixes (and Because of the hotfix version, any SQL scripts in "develop" should be renamed (from "5.11.0" to "5.11.1" for example). To read more about our naming conventions for SQL scripts, see :doc:`sql-upgrade-scripts`. Please note that version bumps and SQL script renaming both require all open pull requests to be updated with the latest from the "develop" branch so you might want to add any SQL script renaming to the hotfix branch before you put it through QA to be merged with develop. This way, open pull requests only need to be updated once. - ----- - -Previous: :doc:`containers` | Next: :doc:`tools` diff --git a/doc/sphinx-guides/source/developers/remote-users.rst b/doc/sphinx-guides/source/developers/remote-users.rst index 484abe9ccf0..38b3edab772 100755 --- a/doc/sphinx-guides/source/developers/remote-users.rst +++ b/doc/sphinx-guides/source/developers/remote-users.rst @@ -75,7 +75,3 @@ In case you want to stop and remove the Keycloak container, just run the other a ``./rm-keycloak.sh`` Note: the Keycloak admin to login at the admin console is ``kcadmin:kcpassword`` - ----- - -Previous: :doc:`unf/index` | Next: :doc:`geospatial` diff --git a/doc/sphinx-guides/source/developers/selinux.rst b/doc/sphinx-guides/source/developers/selinux.rst index dcbf3ee594f..ca41ab82d25 100644 --- a/doc/sphinx-guides/source/developers/selinux.rst +++ b/doc/sphinx-guides/source/developers/selinux.rst @@ -109,7 +109,3 @@ Once your updated SELinux rules are in place, try logging in with Shibboleth aga Keep iterating until it works and then create a pull request based on your updated file. Good luck! Many thanks to Bill Horka from IQSS for his assistance in explaining how to construct a SELinux Type Enforcement (TE) file! - ----- - -Previous: :doc:`geospatial` diff --git a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst index 4689aeec0f2..6ac3fdcb3d6 100644 --- a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst +++ b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst @@ -43,7 +43,3 @@ Renaming SQL Upgrade Scripts Please note that if you need to rename your script (because a new version of the Dataverse Software was released, for example), you will see the error "FlywayException: Validate failed: Detected applied migration not resolved locally" when you attempt to deploy and deployment will fail. To resolve this problem, delete the old migration from the ``flyway_schema_history`` table and attempt to redeploy. - ----- - -Previous: :doc:`version-control` | Next: :doc:`testing` diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst index 24e14ade932..2ea85913d42 100755 --- a/doc/sphinx-guides/source/developers/testing.rst +++ b/doc/sphinx-guides/source/developers/testing.rst @@ -273,7 +273,7 @@ Remember, it’s only a test (and it's not graded)! Some guidelines to bear in m - If you need to test an optional service (S3, etc.), add it to our docker compose file. See :doc:`/container/dev-usage`. - Just as with any development, if you’re stuck: ask for help! -To execute existing integration tests on your local Dataverse installation, a helpful command line tool to use is `Maven `_. You should have Maven installed as per the `Development Environment `_ guide, but if not it’s easily done via Homebrew: ``brew install maven``. +To execute existing integration tests on your local Dataverse installation from the command line, use Maven. You should have Maven installed as per :doc:`dev-environment`, but if not it's easily done via Homebrew: ``brew install maven``. Once installed, you may run commands with ``mvn [options] [] []``. @@ -555,7 +555,3 @@ Future Work on Accessibility Testing ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Using https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible and hooks available from accessibility testing tools, automate the running of accessibility tools on PRs so that developers will receive quicker feedback on proposed code changes that reduce the accessibility of the application. - ----- - -Previous: :doc:`sql-upgrade-scripts` | Next: :doc:`documentation` diff --git a/doc/sphinx-guides/source/developers/tips.rst b/doc/sphinx-guides/source/developers/tips.rst index 764434d1896..839ae3aa19d 100755 --- a/doc/sphinx-guides/source/developers/tips.rst +++ b/doc/sphinx-guides/source/developers/tips.rst @@ -2,7 +2,7 @@ Tips ==== -If you just followed the steps in :doc:`dev-environment` for the first time, you will need to get set up to deploy code to your app server. Below you'll find other tips as well. +If you just followed the steps in :doc:`classic-dev-env` for the first time, you will need to get set up to deploy code to your app server. Below you'll find other tips as well. .. contents:: |toctitle| :local: @@ -10,7 +10,7 @@ If you just followed the steps in :doc:`dev-environment` for the first time, you Iterating on Code and Redeploying --------------------------------- -When you followed the steps in the :doc:`dev-environment` section, the war file was deployed to Payara by the Dataverse Software installation script. That's fine but once you're ready to make a change to the code you will need to get comfortable with undeploying and redeploying code (a war file) to Payara. +When you followed the steps in the :doc:`classic-dev-env` section, the war file was deployed to Payara by the Dataverse Software installation script. That's fine but once you're ready to make a change to the code you will need to get comfortable with undeploying and redeploying code (a war file) to Payara. It's certainly possible to manage deployment and undeployment of the war file via the command line using the ``asadmin`` command that ships with Payara (that's what the Dataverse Software installation script uses and the steps are documented below), but we recommend getting set up with an IDE such as Netbeans to manage deployment for you. @@ -99,7 +99,7 @@ With over 100 tables, the Dataverse Software PostgreSQL database ("dvndb") can b pgAdmin ~~~~~~~~ -Back in the :doc:`dev-environment` section, we had you install pgAdmin, which can help you explore the tables and execute SQL commands. It's also listed in the :doc:`tools` section. +Back in the :doc:`classic-dev-env` section, we had you install pgAdmin, which can help you explore the tables and execute SQL commands. It's also listed in the :doc:`tools` section. SchemaSpy ~~~~~~~~~ @@ -239,7 +239,3 @@ with the following code in ``SettingsWrapper.java``: A more serious example would be direct calls to PermissionServiceBean methods used in render logic expressions. This is something that has happened and caused some problems in real life. A simple permission service lookup (for example, whether a user is authorized to create a dataset in the current dataverse) can easily take 15 database queries. Repeated multiple times, this can quickly become a measurable delay in rendering the page. PermissionsWrapper must be used exclusively for any such lookups from JSF pages. See also :doc:`performance`. - ----- - -Previous: :doc:`dev-environment` | Next: :doc:`troubleshooting` diff --git a/doc/sphinx-guides/source/developers/tools.rst b/doc/sphinx-guides/source/developers/tools.rst index 9d2740fab6a..9b3e38232e8 100755 --- a/doc/sphinx-guides/source/developers/tools.rst +++ b/doc/sphinx-guides/source/developers/tools.rst @@ -2,11 +2,16 @@ Tools ===== -These are handy tools for your :doc:`/developers/dev-environment/`. +These are handy tools for your :doc:`dev-environment`. .. contents:: |toctitle| :local: +Tools for Faster Deployment ++++++++++++++++++++++++++++ + +See :ref:`ide-trigger-code-deploy` in the Container Guide. + Netbeans Connector Chrome Extension +++++++++++++++++++++++++++++++++++ @@ -18,7 +23,7 @@ Unfortunately, while the Netbeans Connector Chrome Extension used to "just work" pgAdmin +++++++ -You probably installed pgAdmin when following the steps in the :doc:`dev-environment` section but if not, you can download it from https://www.pgadmin.org +You may have installed pgAdmin when following the steps in the :doc:`classic-dev-env` section but if not, you can download it from https://www.pgadmin.org Maven +++++ @@ -261,10 +266,3 @@ We can see that the first ``FGC`` resulted in reducing the ``"O"`` by almost 7GB etc. ... It is clearly growing - so now we can conclude that indeed something there is using memory in a way that's not recoverable, and this is a clear problem. - - - - ----- - -Previous: :doc:`making-releases` | Next: :doc:`unf/index` diff --git a/doc/sphinx-guides/source/developers/troubleshooting.rst b/doc/sphinx-guides/source/developers/troubleshooting.rst index 832785f9860..2c437ca8b2e 100755 --- a/doc/sphinx-guides/source/developers/troubleshooting.rst +++ b/doc/sphinx-guides/source/developers/troubleshooting.rst @@ -2,7 +2,7 @@ Troubleshooting =============== -Over in the :doc:`dev-environment` section we described the "happy path" of when everything goes right as you set up your Dataverse Software development environment. Here are some common problems and solutions for when things go wrong. +Over in the :doc:`classic-dev-env` section we described the "happy path" of when everything goes right as you set up your Dataverse Software development environment. Here are some common problems and solutions for when things go wrong. .. contents:: |toctitle| :local: @@ -110,7 +110,3 @@ If you are seeing ``Response code: 400, [url] domain of URL is not allowed`` it' ``./asadmin delete-jvm-options '-Ddataverse.siteUrl=http\://localhost\:8080'`` ``./asadmin create-jvm-options '-Ddataverse.siteUrl=http\://demo.dataverse.org'`` - ----- - -Previous: :doc:`tips` | Next: :doc:`version-control` diff --git a/doc/sphinx-guides/source/developers/unf/index.rst b/doc/sphinx-guides/source/developers/unf/index.rst index 856de209e82..596bb0cf3bf 100644 --- a/doc/sphinx-guides/source/developers/unf/index.rst +++ b/doc/sphinx-guides/source/developers/unf/index.rst @@ -37,7 +37,3 @@ Learn more: Micah Altman and Gary King. 2007. “A Proposed Standard for the Sch unf-v3 unf-v5 unf-v6 - ----- - -Previous: :doc:`/developers/tools` | Next: :doc:`/developers/remote-users` diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst index 85ae62f59a6..c5669d02e77 100644 --- a/doc/sphinx-guides/source/developers/version-control.rst +++ b/doc/sphinx-guides/source/developers/version-control.rst @@ -286,7 +286,3 @@ GitHub documents how to make changes to a fork at https://help.github.com/articl vim path/to/file.txt git commit git push OdumInstitute 4709-postgresql_96 - ----- - -Previous: :doc:`troubleshooting` | Next: :doc:`sql-upgrade-scripts` diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 6eab84092ed..42702bff155 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -13,12 +13,14 @@ services: DATAVERSE_DB_PASSWORD: secret DATAVERSE_DB_USER: ${DATAVERSE_DB_USER} ENABLE_JDWP: "1" + ENABLE_RELOAD: "1" + SKIP_DEPLOY: "${SKIP_DEPLOY}" + DATAVERSE_JSF_REFRESH_PERIOD: "1" DATAVERSE_FEATURE_API_BEARER_AUTH: "1" DATAVERSE_AUTH_OIDC_ENABLED: "1" DATAVERSE_AUTH_OIDC_CLIENT_ID: test DATAVERSE_AUTH_OIDC_CLIENT_SECRET: 94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8 DATAVERSE_AUTH_OIDC_AUTH_SERVER_URL: http://keycloak.mydomain.com:8090/realms/test - DATAVERSE_JSF_REFRESH_PERIOD: "1" # These two oai settings are here to get HarvestingServerIT to pass dataverse_oai_server_maxidentifiers: "2" dataverse_oai_server_maxrecords: "2" @@ -48,7 +50,7 @@ services: -Ddataverse.files.minio1.secret-key=s3cr3t_4cc355_k3y ports: - "8080:8080" # HTTP (Dataverse Application) - - "4848:4848" # HTTP (Payara Admin Console) + - "4949:4848" # HTTPS (Payara Admin Console) - "9009:9009" # JDWP - "8686:8686" # JMX networks: @@ -60,11 +62,7 @@ services: volumes: - ./docker-dev-volumes/app/data:/dv - ./docker-dev-volumes/app/secrets:/secrets - # Uncomment to map the glassfish applications folder so that we can update webapp resources using scripts/intellij/cpwebapp.sh - # - ./docker-dev-volumes/glassfish/applications:/opt/payara/appserver/glassfish/domains/domain1/applications - # Uncomment for changes to xhtml to be deployed immediately (if supported your IDE or toolchain). - # Replace 6.0 with the current version. - # - ./target/dataverse-6.0:/opt/payara/deployments/dataverse + - ./target/dataverse:/opt/payara/deployments/dataverse:ro tmpfs: - /dumps:mode=770,size=2052M,uid=1000,gid=1000 - /tmp:mode=770,size=2052M,uid=1000,gid=1000 @@ -176,12 +174,21 @@ services: volumes: - './conf/keycloak/test-realm.json:/opt/keycloak/data/import/test-realm.json' - dev_nginx: - container_name: dev_nginx - image: gdcc/dev_nginx:unstable + # This proxy configuration is only intended to be used for development purposes! + # DO NOT USE IN PRODUCTION! HIGH SECURITY RISK! + dev_proxy: + image: caddy:2-alpine + # The command below is enough to enable using the admin gui, but it will not rewrite location headers to HTTP. + # To achieve rewriting from https:// to http://, we need a simple configuration file + #command: ["caddy", "reverse-proxy", "-f", ":4848", "-t", "https://dataverse:4848", "--insecure"] + command: ["caddy", "run", "-c", "/Caddyfile"] ports: - - "4849:4849" + - "4848:4848" # Will expose Payara Admin Console (HTTPS) as HTTP restart: always + volumes: + - ./conf/proxy/Caddyfile:/Caddyfile:ro + depends_on: + - dev_dataverse networks: - dataverse diff --git a/docker/util/intellij/README.md b/docker/util/intellij/README.md new file mode 100644 index 00000000000..281d0e50ea6 --- /dev/null +++ b/docker/util/intellij/README.md @@ -0,0 +1,13 @@ +# IntelliJ Auto-Copy of Webapp Files + +When deploying the webapp via Payara Tools, you can use this tool to immediately copy changes to non-code files into the running deployment, instantly seeing changes in your browser. + +Note: as this relies on using a Bash shell script, it is pretty much limited to Mac and Linux. +Feel free to extend and provide a PowerShell equivalent! + +1. Install the [File Watcher plugin](https://plugins.jetbrains.com/plugin/7177-file-watchers) +2. Import the [watchers.xml](./watchers.xml) file at *File > Settings > Tools > File Watchers* +3. Once you have the deployment running (see Container Guides), editing files at `src/main/webapp` will be copied into the deployment after saving the edited file. + +Alternatively, you can add an External tool and trigger via menu or shortcut to do the copying manually: +https://www.jetbrains.com/help/idea/configuring-third-party-tools.html diff --git a/docker/util/intellij/cpwebapp.sh b/docker/util/intellij/cpwebapp.sh new file mode 100755 index 00000000000..2d08fb1a873 --- /dev/null +++ b/docker/util/intellij/cpwebapp.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# cpwebapp +# + +set -eu + +PROJECT_DIR="$1" +FILE_TO_COPY="$2" +RELATIVE_PATH="${FILE_TO_COPY#"${PROJECT_DIR}/"}" + +# Check if RELATIVE_PATH starts with 'src/main/webapp', otherwise ignore +if [[ "$RELATIVE_PATH" == "src/main/webapp"* ]]; then + # Extract version from POM, so we don't need to have Maven on the PATH + VERSION=$(grep -oPm1 "(?<=)[^<]+" "$PROJECT_DIR/modules/dataverse-parent/pom.xml") + + # Construct the target path by cutting off the local prefix and prepend with in-container path + RELATIVE_PATH_WITHOUT_WEBAPP="${RELATIVE_PATH#src/main/webapp/}" + TARGET_PATH="/opt/payara/appserver/glassfish/domains/domain1/applications/dataverse-$VERSION/${RELATIVE_PATH_WITHOUT_WEBAPP}" + + # Copy file to container + docker cp "$FILE_TO_COPY" "dev_dataverse:$TARGET_PATH" +fi diff --git a/scripts/intellij/watchers.xml b/docker/util/intellij/watchers.xml similarity index 90% rename from scripts/intellij/watchers.xml rename to docker/util/intellij/watchers.xml index e118fea558f..4ccee125ec2 100644 --- a/scripts/intellij/watchers.xml +++ b/docker/util/intellij/watchers.xml @@ -12,7 +12,7 @@