diff --git a/src/exportMap/builder.js b/src/exportMap/builder.js
index 5348dba37..f7b9006ef 100644
--- a/src/exportMap/builder.js
+++ b/src/exportMap/builder.js
@@ -92,7 +92,11 @@ export default class ExportMapBuilder {
 
     exportMap.mtime = stats.mtime;
 
-    exportCache.set(cacheKey, exportMap);
+    // If the visitor keys were not populated, then we shouldn't save anything to the cache,
+    // since the parse results may not be reliable.
+    if (exportMap.visitorKeys) {
+      exportCache.set(cacheKey, exportMap);
+    }
     return exportMap;
   }
 
diff --git a/tests/src/core/getExports.js b/tests/src/core/getExports.js
index 76003410d..73fa4e950 100644
--- a/tests/src/core/getExports.js
+++ b/tests/src/core/getExports.js
@@ -11,6 +11,12 @@ import * as fs from 'fs';
 import { getFilename } from '../utils';
 import { test as testUnambiguous } from 'eslint-module-utils/unambiguous';
 
+const doesBabelHaveVisitorKeys = () => {
+  const babelPath = require.resolve('babel-eslint');
+  const hypotheticalLocation = babelPath.replace('index.js', 'visitor-keys.js');
+  return fs.existsSync(hypotheticalLocation);
+};
+
 describe('ExportMap', function () {
   const fakeContext = Object.assign(
     semver.satisfies(eslintPkg.version, '>= 7.28') ? {
@@ -21,10 +27,12 @@ describe('ExportMap', function () {
     },
     {
       settings: {},
-      parserPath: 'babel-eslint',
+      parserPath: require.resolve('babel-eslint'),
     },
   );
 
+  const isVisitorKeysSupported = doesBabelHaveVisitorKeys();
+
   it('handles ExportAllDeclaration', function () {
     let imports;
     expect(function () {
@@ -36,11 +44,20 @@ describe('ExportMap', function () {
 
   });
 
-  it('returns a cached copy on subsequent requests', function () {
+  isVisitorKeysSupported && it('returns a cached copy on subsequent requests', function () {
     expect(ExportMapBuilder.get('./named-exports', fakeContext))
       .to.exist.and.equal(ExportMapBuilder.get('./named-exports', fakeContext));
   });
 
+  it('does not return a cached copy if the parse does not yield a visitor keys', function () {
+    const mockContext = {
+      ...fakeContext,
+      parserPath: 'not-real',
+    };
+    expect(ExportMapBuilder.get('./named-exports', mockContext))
+      .to.exist.and.not.equal(ExportMapBuilder.get('./named-exports', mockContext));
+  });
+
   it('does not return a cached copy after modification', (done) => {
     const firstAccess = ExportMapBuilder.get('./mutator', fakeContext);
     expect(firstAccess).to.exist;