[docs] Clean up doc structure (first part) (ray-project#21667)

.gitignore

@@ -115,10 +115,7 @@ scripts/nodes.txt
 **/MNIST_data/
 **/cifar-10-batches-bin/
 
-# Generated documentation files
-/doc/_build
-/doc/source/_static/thumbs
-/doc/source/tune/generated_guides/
+
 
 # User-specific stuff:
 .idea/**/workspace.xml

doc/.gitignore

@@ -1 +1,8 @@
-auto_examples/
+# Generated documentation files
+_build
+source/_static/thumbs
+
+source/ray-core/examples/
+source/ray-tune/tutorials/
+source/ray-tune/generated_guides/
+source/ray-data/examples/

doc/BUILD

@@ -8,8 +8,8 @@
 py_test(
     name = "dask_xgboost",
     size = "medium",
-    main = "examples/dask_xgboost/dask_xgboost.py",
-    srcs = ["examples/dask_xgboost/dask_xgboost.py"],
+    main = "source/ray-core/_examples/dask_xgboost/dask_xgboost.py",
+    srcs = ["source/ray-core/_examples/dask_xgboost/dask_xgboost.py"],
     tags = ["exclusive", "team:ml", "py37"],
     args = ["--smoke-test", "--address ''", "--num-actors 4",
             "--cpus-per-actor 1", "--num-actors-inference 4",
@@ -20,8 +20,8 @@ py_test(
 py_test(
     name = "modin_xgboost",
     size = "medium",
-    main = "examples/modin_xgboost/modin_xgboost.py",
-    srcs = ["examples/modin_xgboost/modin_xgboost.py"],
+    main = "source/ray-core/_examples/modin_xgboost/modin_xgboost.py",
+    srcs = ["source/ray-core/_examples/modin_xgboost/modin_xgboost.py"],
     tags = ["exclusive", "team:ml", "py37"],
     args = ["--smoke-test", "--address ''", "--num-actors 4",
             "--cpus-per-actor 1", "--num-actors-inference 4",
@@ -31,77 +31,77 @@ py_test(
 py_test(
     name = "big_data_ingestion",
     size = "small",
-    srcs = ["source/data/_examples/big_data_ingestion.py"],
+    srcs = ["source/ray-data/_examples/big_data_ingestion.py"],
     tags = ["exclusive", "team:core", "py37"]
 )
 
 py_test(
     name = "datasets_train",
     size = "medium",
-    srcs = ["examples/datasets_train/datasets_train.py"],
+    srcs = ["source/ray-core/_examples/datasets_train/datasets_train.py"],
     tags = ["exclusive", "team:ml", "py37", "datasets_train"],
     args = ["--smoke-test", "--num-workers=2", "--use-gpu"]
 )
 
 py_test(
     name = "plot_hyperparameter",
     size = "small",
-    srcs = ["examples/plot_hyperparameter.py"],
+    srcs = ["source/ray-core/_examples/plot_hyperparameter.py"],
     tags = ["exclusive", "team:ml"]
 )
 
 py_test(
     name = "plot_parameter_server",
     size = "medium",
-    srcs = ["examples/plot_parameter_server.py"],
+    srcs = ["source/ray-core/_examples/plot_parameter_server.py"],
     tags = ["exclusive", "team:ml"]
 )
 
 py_test(
     name = "plot_pong_example",
     size = "large",
-    srcs = ["examples/plot_pong_example.py"],
+    srcs = ["source/ray-core/_examples/plot_pong_example.py"],
     tags = ["exclusive", "team:ml"]
 )
 
 py_test(
     name = "progress_bar",
     size = "small",
-    srcs = ["examples/progress_bar.py"],
+    srcs = ["source/ray-core/_examples/progress_bar.py"],
     tags = ["exclusive", "team:ml"]
 )
 
 
-# Directory: examples/doc_code
+# Directory: source/ray-core/_examples/doc_code
 py_test(
     name = "doc_code_tf_example",
     size = "small",
-    main = "examples/doc_code/tf_example.py",
-    srcs = ["examples/doc_code/tf_example.py"],
+    main = "source/ray-core/_examples/doc_code/tf_example.py",
+    srcs = ["source/ray-core/_examples/doc_code/tf_example.py"],
     tags = ["exclusive", "tf", "team:ml"]
 )
 
 py_test(
     name = "doc_code_torch_example",
     size = "small",
-    main = "examples/doc_code/torch_example.py",
-    srcs = ["examples/doc_code/torch_example.py"],
+    main = "source/ray-core/_examples/doc_code/torch_example.py",
+    srcs = ["source/ray-core/_examples/doc_code/torch_example.py"],
     tags = ["exclusive", "pytorch", "team:ml"]
 )
 
 py_test(
     name = "doc_code_metrics_example",
     size = "small",
-    main = "examples/doc_code/metrics_example.py",
-    srcs = ["examples/doc_code/metrics_example.py"],
+    main = "source/ray-core/_examples/doc_code/metrics_example.py",
+    srcs = ["source/ray-core/_examples/doc_code/metrics_example.py"],
     tags = ["exclusive", "team:serve"]
 )
 
 py_test(
     name = "doc_code_runtime_env_example",
     size = "small",
-    main = "examples/doc_code/runtime_env_example.py",
-    srcs = ["examples/doc_code/runtime_env_example.py"],
+    main = "source/ray-core/_examples/doc_code/runtime_env_example.py",
+    srcs = ["source/ray-core/_examples/doc_code/runtime_env_example.py"],
     tags = ["exclusive", "post_wheel_build", "team:serve"]
 )
 
@@ -113,17 +113,17 @@ py_test(
 py_test(
     name = "tune_sklearn",
     size = "medium",
-    main = "source/tune/_tutorials/tune-sklearn.py",
-    srcs = ["source/tune/_tutorials/tune-sklearn.py"],
+    main = "source/ray-tune/_tutorials/tune-sklearn.py",
+    srcs = ["source/ray-tune/_tutorials/tune-sklearn.py"],
     tags = ["exclusive", "example", "team:ml"],
     args = ["--smoke-test"]
 )
 
 py_test(
     name = "tune_serve_integration_mnist",
     size = "medium",
-    main = "source/tune/_tutorials/tune-serve-integration-mnist.py",
-    srcs = ["source/tune/_tutorials/tune-serve-integration-mnist.py"],
+    main = "source/ray-tune/_tutorials/tune-serve-integration-mnist.py",
+    srcs = ["source/ray-tune/_tutorials/tune-serve-integration-mnist.py"],
     tags = ["exclusive", "example", "team:ml"],
     args = ["--smoke-test", "--from_scratch", "--day 0"]
 )

doc/Makefile

@@ -6,7 +6,10 @@ SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
 PAPER         =
 BUILDDIR      = _build
-AUTOGALLERYDIR= source/auto_examples source/tune/tutorials source/tune/generated_guides source/data/examples
+AUTOGALLERYDIR= source/ray-core/examples \
+                source/ray-tune/tutorials \
+                source/ray-tune/generated_guides \
+                source/ray-data/examples
 
 # User-friendly check for sphinx-build
 ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
@@ -54,7 +57,7 @@ clean:
 	rm -rf site/_site site/Gemfile.lock site/.sass-cache
 
 html:
-	$(SPHINXBUILD) -W -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
@@ -173,6 +176,7 @@ linkcheck:
 	@echo "Link check complete; look for any errors in the above output " \
 	      "or in $(BUILDDIR)/linkcheck/output.txt."
 
+# Currently most (>98%) of doctests fail.
 doctest:
 	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
 	@echo "Testing of doctests in the sources finished, look at the " \

doc/README.md

@@ -1,24 +1,77 @@
 # Ray Documentation
 
-To compile the documentation, run the following commands from this directory.
-Note that Ray must be installed first.
+Repository for documentation of the Ray project, hosted at [docs.ray.io](https://docs.ray.io).
+
+## Installation
+
+To build the documentation, make sure you have `ray` installed first.
+For building the documentation locally, allowing for faster builds, install the _development_ dependencies:
+
+```bash
+pip install -r requirements-dev.txt  # development dependencies for faster builds
+```
+
+If you want to reproduce the production environment and its build, install the _production_ dependencies instead:
+
+
+```bash
+pip install -r requirements-doc.txt  # readthedocs.org dependencies
+````
+
+## Building the documentation
+
+To compile the documentation and open it locally, run the following command from this directory.
 
 ```bash
-pip install -r requirements-doc.txt
-pip install -U -r requirements-rtd.txt # important for reproducing the deployment environment
-make html
-open _build/html/index.html
+make html && open _build/html/index.html
 ```
 
-To test if there are any build errors with the documentation, do the following.
+To build the documentation more strictly, by treating warnings as errors, run the following command
+(the `-W` flag is required for this to work):
 
 ```bash
 sphinx-build -W -b html -d _build/doctrees source _build/html
 ```
 
+## Building just one sub-project
+
+Often your changes in documentation just concern one sub-project, such as Tune or Train. 
+To build just this one sub-project, and ignore the rest (leading to build warnings due to broken references etc.), run the following command:
+
+```shell
+DOC_LIB=<project> sphinx-build -b html -d _build/doctrees  source _build/html
+```
+where `<project>` is the name of the sub-project and can be any of "cluster", "contribute",
+"ray-core", "ray-data", "ray-design-patterns", "ray-more-libs",
+"ray-observability", "ray-overview", "ray-rllib",  "ray-serve", "ray-train",
+"ray-tune", or "ray-workflows".
+
+## Announcements and includes
+
+To add new announcements and other messaging to the top or bottom of a documentation page,
+check the `_includes` folder first to see if the message you want is already there (like "get help" 
+or "we're hiring" etc.)
+If not, add the template you want and include it accordingly, i.e. with
+
+```markdown
+.. include:: /_includes/<my-announcement>
+```
+
+This ensures consistent messaging across documentation pages.
+
+## Checking for broken links
+
 To check if there are broken links, run the following (we are currently not running this
 in the CI since there are false positives).
 
 ```bash
 make linkcheck
 ```
+
+## Running doctests
+
+To run tests for examples shipping with docstrings in Python files, run the following command:
+
+```shell
+make doctest
+```

doc/requirements-dev.txt

@@ -0,0 +1,43 @@
+click
+colorama
+colorful
+filelock
+flask
+flatbuffers
+fastapi
+jsonschema
+mock
+numpy
+scikit-image
+pandas
+pickle5
+pillow
+pyarrow
+pydantic
+pygments
+pyyaml
+redis
+sphinx==4.3.2
+sphinx-click
+sphinx-copybutton
+sphinxemoji
+sphinx-gallery
+sphinx-jsonschema
+sphinx-tabs==3.2.0
+sphinx-version-warning
+sphinx-book-theme==0.1.7
+sphinx-external-toc==0.2.3
+sphinxcontrib.yt
+starlette
+tabulate
+uvicorn
+werkzeug
+git+https://github.com/ray-project/tune-sklearn@master#tune-sklearn
+git+https://github.com/ray-project/xgboost_ray@master#egg=xgboost_ray
+git+https://github.com/ray-project/lightgbm_ray@main#lightgbm_ray
+git+https://github.com/ray-project/ray_lightning#ray_lightning
+scikit-optimize
+sphinx-sitemap==2.2.0
+myst-parser
+myst-nb
+pytest

doc/requirements-doc.txt

@@ -14,7 +14,7 @@ pickle5
 pillow
 pyarrow
 pydantic
-pygments
+Pygments==2.3.1
 pyyaml
 recommonmark
 redis
@@ -27,6 +27,7 @@ sphinx-jsonschema
 sphinx-tabs
 sphinx-version-warning
 sphinx-book-theme==0.0.42
+sphinx-external-toc==0.2.3
 sphinxcontrib.yt
 starlette
 tabulate
@@ -39,3 +40,4 @@ git+https://github.com/ray-project/ray_lightning#ray_lightning
 scikit-optimize
 sphinx-sitemap==2.2.0
 myst-parser
+myst-nb

doc/requirements-rtd.txt

@@ -1,13 +1,43 @@
+click
+colorama
+colorful
+filelock
+flask
+flatbuffers
+fastapi
+jsonschema
+mock
+numpy
+scikit-image
+pandas
+pickle5
+pillow
+pyarrow
+pydantic
 Pygments==2.3.1
-setuptools==41.0.1
-docutils==0.16
-mock==1.0.1
-pillow==8.3.2
-alabaster>=0.7,<0.8,!=0.7.5
-commonmark==0.8.1
-recommonmark==0.5.0
+pyyaml
+recommonmark
+redis
 sphinx==3.0.4
-readthedocs-sphinx-ext<1.1
+sphinx-click
+sphinx-copybutton
+sphinxemoji
+sphinx-gallery
+sphinx-jsonschema
+sphinx-tabs
+sphinx-version-warning
 sphinx-book-theme==0.0.42
+sphinx-external-toc==0.2.3
+sphinxcontrib.yt
+starlette
+tabulate
+uvicorn==0.16.0
+werkzeug
+git+https://github.com/ray-project/tune-sklearn@master#tune-sklearn
+git+https://github.com/ray-project/xgboost_ray@master#egg=xgboost_ray
+git+https://github.com/ray-project/lightgbm_ray@main#lightgbm_ray
+git+https://github.com/ray-project/ray_lightning#ray_lightning
+scikit-optimize
 sphinx-sitemap==2.2.0
 myst-parser
+myst-nb