build: Clear out stale dependencies when installing requirements (#31080

)

We've been seeing some cross-version caching issues in the
edx-platform-runner unit tests, which apparently run on a "dirty"
environment—the virtualenv does not get cleared out between runs,
which probably improves performance but allows installed dependencies
to "leak" between runs. This results in errors between master and older
open-releases but could also prevent us from noticing missing deps.

By using pip-sync in the new CI Make targets (as we already do for
the regular `make requirements` target) we can ensure that any stale
dependencies from runs by other branches (or older versions of the
code) are removed.

Calling `make local-requirements` at the end of each `*-requirements`
target rather than making it a prerequisite is necessary for using sync,
since otherwise the local reqs would be wiped out.

The `requirements` target is also deduplicated into the newer
`dev-requirements` (aliased to it, with both installing private deps now.)

Adding a prerequisite of `pre-requirements` allows us to simplify some
workflow calls slightly. This ends up being the bulk of the commit by
line count. The pip lockfile also wasn't being used in the Makefile, so
I added that to pre-requirements as well.

Also fix leading whitespace issue in Makefile.

.github/workflows/js-tests.yml

@@ -63,7 +63,6 @@ jobs:
 
     - name: Install Required Python Dependencies
       run: |
-        pip install -r requirements/pip.txt
         make base-requirements
 
     - uses: c-hive/gha-npm-cache@v1

.github/workflows/migrations-check-mysql8.yml

@@ -50,7 +50,6 @@ jobs:
 
     - name: Install Python dependencies
       run: |
-        pip install -r requirements/pip.txt
         make dev-requirements
         pip uninstall -y mysqlclient
         pip install --no-binary mysqlclient mysqlclient

.github/workflows/migrations-check.yml

@@ -60,7 +60,6 @@ jobs:
 
     - name: Install Python dependencies
       run: |
-        pip install -r requirements/pip.txt
         make dev-requirements
         pip uninstall -y mysqlclient
         pip install --no-binary mysqlclient mysqlclient

.github/workflows/pylint-checks.yml

@@ -57,7 +57,6 @@ jobs:
 
       - name: Install required Python dependencies
         run: |
-          pip install -r requirements/pip.txt
           make dev-requirements
           pip uninstall -y mysqlclient
           pip install --no-binary mysqlclient mysqlclient

.github/workflows/static-assets-check.yml

@@ -49,7 +49,6 @@ jobs:
 
     - name: Install Required Python Dependencies
       run: |
-        pip install -r requirements/pip.txt
         make base-requirements
 
     - name: Initiate Mongo DB Service

.github/workflows/unit-tests.yml

@@ -61,7 +61,6 @@ jobs:
 
       - name: install requirements
         run: |
-          sudo pip install -r requirements/pip.txt
           sudo make test-requirements
           if [[ "${{ matrix.django-version }}" == "pinned" ]]; then
             sudo pip install -r requirements/edx/django.txt

.github/workflows/verify-gha-unit-tests-count.yml

@@ -17,7 +17,6 @@ jobs:
       - uses: actions/checkout@v2
       - name: install requirements
         run: |
-          sudo pip install -r requirements/pip.txt
           sudo make test-requirements
 
       - name: verify unit tests count

Makefile

@@ -64,26 +64,28 @@ pull: ## update the Docker image used by "make shell"
 	docker pull edxops/edxapp:latest
 
 pre-requirements: ## install Python requirements for running pip-tools
+	pip install -qr requirements/pip.txt
 	pip install -qr requirements/edx/pip-tools.txt
 
 local-requirements:
 # 	edx-platform installs some Python projects from within the edx-platform repo itself.
 	pip install -e .
 
-dev-requirements: local-requirements
-	pip install -qr requirements/edx/development.txt
-
-base-requirements: local-requirements
-	pip install -qr requirements/edx/base.txt
-
-test-requirements: local-requirements
-	pip install --exists-action='w' -qr requirements/edx/testing.txt
-
-requirements: pre-requirements ## install development environment requirements
+dev-requirements: pre-requirements
 	@# The "$(wildcard..)" is to include private.txt if it exists, and make no mention
 	@# of it if it does not.  Shell wildcarding can't do that with default options.
 	pip-sync -q requirements/edx/development.txt $(wildcard requirements/edx/private.txt)
-	pip install -e .
+	make local-requirements
+
+base-requirements: pre-requirements
+	pip-sync requirements/edx/base.txt
+	make local-requirements
+
+test-requirements: pre-requirements
+	pip-sync --pip-args="--exists-action=w" requirements/edx/testing.txt
+	make local-requirements
+
+requirements: dev-requirements ## install development environment requirements
 
 shell: ## launch a bash shell in a Docker container with all edx-platform dependencies installed
 	docker run -it -e "NO_PYTHON_UNINSTALL=1" -e "PIP_INDEX_URL=https://pypi.python.org/simple" -e TERM \
@@ -130,7 +132,7 @@ compile-requirements: $(COMMON_CONSTRAINTS_TXT) ## Re-compile *.in requirements
 		pip-compile -v --no-emit-trusted-host --no-emit-index-url $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in || exit 1; \
 		export REBUILD=''; \
 	done
- 	# Let tox control the Django version for tests
+	# Let tox control the Django version for tests
 	grep -e "^django==" requirements/edx/base.txt > requirements/edx/django.txt
 	sed '/^[dD]jango==/d' requirements/edx/testing.txt > requirements/edx/testing.tmp
 	mv requirements/edx/testing.tmp requirements/edx/testing.txt