Skip to content

Commit

Permalink
Accelerate refresh during sync, improve progress reporting
Browse files Browse the repository at this point in the history 
Especially on large workspaces the initial Eclipse refresh can take an
extreme long time. This replaces the full refresh with a local version
doing much less work.
  • Loading branch information
@guw
guw committed Aug 29, 2023
1 parent 5122366 commit bcdd293
Show file tree
Hide file tree
Showing 7 changed files with 129 additions and 79 deletions.
24 changes: 15 additions & 9 deletions ...l.eclipse.core/src/com/salesforce/bazel/eclipse/core/classpath/BazelClasspathManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
import static java.util.Objects.requireNonNull;
import static java.util.function.Predicate.not;
import static java.util.stream.Collectors.toList;
import static org.eclipse.core.runtime.SubMonitor.SUPPRESS_ALL_LABELS;
import static org.eclipse.core.runtime.SubMonitor.SUPPRESS_NONE;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
Expand Down Expand Up @@ -236,7 +238,7 @@ TargetProvisioningStrategy getTargetProvisioningStrategy(BazelWorkspace bazelWor
public void persistAttachedSourcesAndJavadoc(IJavaProject project, IClasspathContainer containerSuggestion,
IProgressMonitor progress) throws CoreException {
try {
var monitor = SubMonitor.convert(progress, 2);
var monitor = SubMonitor.convert(progress, "Saving classpath container for " + project.getElementName(), 2);
var bazelProject = getBazelProject(project);
if (bazelProject == null) {
return;
Expand Down Expand Up @@ -272,7 +274,7 @@ public void persistAttachedSourcesAndJavadoc(IJavaProject project, IClasspathCon
List.of(bazelProject),
bazelProject.getBazelWorkspace(),
DEFAULT_CLASSPATH,
monitor.split(1));
monitor.split(1, SUPPRESS_ALL_LABELS));
entries =
configureClasspathWithSourceAttachments(classpaths.get(bazelProject), null /* no props */, monitor);
for (IClasspathEntry entry : entries) {
Expand Down Expand Up @@ -300,7 +302,10 @@ public void persistAttachedSourcesAndJavadoc(IJavaProject project, IClasspathCon
}

// update classpath container (this will re-set classpath on JavaProject)
updateClasspath(bazelProject.getBazelWorkspace(), List.of(bazelProject), monitor.split(1));
updateClasspath(
bazelProject.getBazelWorkspace(),
List.of(bazelProject),
monitor.split(1, SUPPRESS_ALL_LABELS));
} finally {
if (progress != null) {
progress.done();
Expand Down Expand Up @@ -331,7 +336,7 @@ void saveContainerState(IProject project, IClasspathContainer container) throws
void updateClasspath(BazelWorkspace bazelWorkspace, List<BazelProject> projects, IProgressMonitor progress)
throws CoreException {
try {
var monitor = SubMonitor.convert(progress, 2 + projects.size());
var monitor = SubMonitor.convert(progress, "Computing classpath of Bazel projects", 2 + projects.size());

// we need to refresh the workspace project differently
var workspaceProject = bazelWorkspace.getBazelProject();
Expand All @@ -346,14 +351,15 @@ void updateClasspath(BazelWorkspace bazelWorkspace, List<BazelProject> projects,
.collect(toList());

// ensure the packages are opened efficiently
monitor.subTask("Reading packages...");
bazelWorkspace.open(getBazelPackages(bazelWorkspace, nonWorkspaceProjects));

// compute classpaths for all non-workspace projects
monitor.subTask("Computing classpaths...");
var strategy = getTargetProvisioningStrategy(bazelWorkspace);
var classpaths = strategy
.computeClasspaths(nonWorkspaceProjects, bazelWorkspace, DEFAULT_CLASSPATH, monitor.split(1));
var classpaths = strategy.computeClasspaths(
nonWorkspaceProjects,
bazelWorkspace,
DEFAULT_CLASSPATH,
monitor.split(1, SUPPRESS_NONE));

// apply classpaths for each project
for (BazelProject bazelProject : projects) {
Expand Down Expand Up @@ -381,7 +387,7 @@ void updateClasspath(BazelWorkspace bazelWorkspace, List<BazelProject> projects,
container.getPath(),
new IJavaProject[] { javaProject },
new IClasspathContainer[] { container },
monitor.newChild(1));
monitor.split(1));
saveContainerState(bazelProject.getProject(), container);

}
Expand Down
19 changes: 11 additions & 8 deletions ...core/src/com/salesforce/bazel/eclipse/core/classpath/InitializeOrRefreshClasspathJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import static com.salesforce.bazel.eclipse.core.BazelCoreSharedContstants.BAZEL_NATURE_ID;
import static java.lang.String.format;
import static java.util.stream.Collectors.groupingBy;
import static org.eclipse.core.runtime.SubMonitor.SUPPRESS_NONE;
import static org.eclipse.core.runtime.SubMonitor.convert;

import java.util.Collection;
import java.util.List;
Expand All @@ -20,7 +22,6 @@
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.MultiStatus;
import org.eclipse.core.runtime.Status;
import org.eclipse.core.runtime.SubMonitor;
import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.jdt.core.IJavaModelMarker;
import org.eclipse.jdt.core.JavaCore;
Expand Down Expand Up @@ -135,9 +136,9 @@ boolean needsRefresh(BazelProject p) {
}

@Override
public IStatus runInWorkspace(IProgressMonitor monitor) throws CoreException {
public IStatus runInWorkspace(IProgressMonitor progress) throws CoreException {
try {
var subMonitor = SubMonitor.convert(monitor, bazelProjects.size());
var monitor = convert(progress, "Computing Bazel Classpaths", bazelProjects.size());
var status =
new MultiStatus(BazelModelManager.PLUGIN_ID, 0, "Some Bazel build paths could not be initialized.");

Expand All @@ -148,8 +149,10 @@ public IStatus runInWorkspace(IProgressMonitor monitor) throws CoreException {
continue nextProjectSet;
}

getClasspathManager()
.updateClasspath(projectSet.getKey(), projectSet.getValue(), subMonitor.newChild(1));
getClasspathManager().updateClasspath(
projectSet.getKey(),
projectSet.getValue(),
monitor.split(1, SUPPRESS_NONE));
} catch (CoreException e) {
status.add(e.getStatus());

Expand All @@ -173,16 +176,16 @@ public IStatus runInWorkspace(IProgressMonitor monitor) throws CoreException {
}
); // @formatter:on
}
subMonitor.worked(1);
monitor.worked(1);
}

// return error if we have one!
if (status.matches(IStatus.ERROR)) {
return status;
}
} finally {
if (monitor != null) {
monitor.done();
if (progress != null) {
progress.done();
}
}
return Status.OK_STATUS;
Expand Down
121 changes: 78 additions & 43 deletions ...l.eclipse.core/src/com/salesforce/bazel/eclipse/core/model/SynchronizeProjectViewJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@
import static java.nio.file.Files.isReadable;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
import static org.eclipse.core.resources.IResource.DEPTH_INFINITE;
import static org.eclipse.core.resources.IContainer.INCLUDE_HIDDEN;
import static org.eclipse.core.resources.IResource.DEPTH_ONE;
import static org.eclipse.core.resources.IResource.FORCE;
import static org.eclipse.core.runtime.SubMonitor.SUPPRESS_NONE;

Expand All @@ -31,7 +32,6 @@
import org.eclipse.core.resources.IProject;
import org.eclipse.core.resources.IResource;
import org.eclipse.core.resources.IResourceFilterDescription;
import org.eclipse.core.resources.IResourceVisitor;
import org.eclipse.core.resources.IWorkspace;
import org.eclipse.core.resources.IWorkspaceRoot;
import org.eclipse.core.resources.ResourcesPlugin;
Expand Down Expand Up @@ -137,7 +137,7 @@ public SynchronizeProjectViewJob(BazelWorkspace workspace) throws CoreException
setRule(getWorkspaceRoot());
}

private void configureFiltersAndRefresh(IProject workspaceProject, SubMonitor monitor) throws CoreException {
private void configureFilters(IProject workspaceProject, SubMonitor monitor) throws CoreException {
var filterExists = Stream.of(workspaceProject.getFilters()).anyMatch(f -> {
var matcher = f.getFileInfoMatcherDescription();
return RESOURCE_FILTER_BAZEL_OUTPUT_SYMLINKS_ID.equals(matcher.getId());
Expand All @@ -151,9 +151,6 @@ private void configureFiltersAndRefresh(IProject workspaceProject, SubMonitor mo
new FileInfoMatcherDescription(RESOURCE_FILTER_BAZEL_OUTPUT_SYMLINKS_ID, null),
NONE,
monitor);
} else {
// filter exists, just refresh the project
workspaceProject.refreshLocal(DEPTH_INFINITE, monitor);
}
}

Expand Down Expand Up @@ -364,8 +361,8 @@ IWorkspaceRoot getWorkspaceRoot() {
return getWorkspace().getRoot();
}

private void hideFoldersNotVisibleAccordingToProjectView(IProject workspaceProject, SubMonitor monitor)
throws CoreException {
private void hideFoldersNotVisibleAccordingToProjectViewAndSmartRefresh(IProject workspaceProject,
SubMonitor monitor) throws CoreException {
monitor.beginTask("Configuring visible folders", 10);

// we are comparing using project relative paths
Expand All @@ -384,36 +381,13 @@ private void hideFoldersNotVisibleAccordingToProjectView(IProject workspaceProje
}
}

IResourceVisitor visitor = resource -> {
// we only hide folders, i.e. all files contained in the project remain visible
if (resource.getType() == IResource.FOLDER) {
var path = resource.getProjectRelativePath();
if (findPathOrAnyParentInSet(path, alwaysAllowedFolders)) {
// never hide those in the always allowed folders
resource.setHidden(false);
return false;
}

// we need to check three things
// 1. if workspace root '.' is listed, everything should be visible by default
// 2. if a parent is in visiblePaths it should be visible
// 3. if a sub-sub-sub directory is in importRoots then it should be visible as well
var visible = foundWorkspaceRoot || visiblePaths.contains(resource.getProjectRelativePath())
|| importRoots.containsWorkspacePath(new WorkspacePath(path.toString()));
// but an explicit exclude dominates
var excluded = importRoots.isExcluded(new WorkspacePath(path.toString()));

resource.setHidden(excluded || !visible);
return visible && !excluded; // no need to continue looking if it's not visible
}

// we cannot make a decision, continue searching
return true;
};
workspaceProject.accept(
visitor,
DEPTH_INFINITE,
IContainer.INCLUDE_HIDDEN /* visit hidden ones so we can un-hide if necessary */);
refreshFolderAndHideMembersIfNecessary(
monitor,
alwaysAllowedFolders,
foundWorkspaceRoot,
visiblePaths,
3 /* max deepness */,
workspaceProject);
}

private void importPreferences(Collection<WorkspacePath> importPreferences, SubMonitor monitor)
Expand Down Expand Up @@ -473,6 +447,65 @@ private List<BazelProject> provisionProjectsForTarget(Set<BazelTarget> targets,
return getTargetProvisioningStrategy().provisionProjectsForSelectedTargets(targets, workspace, monitor);
}

private void refreshFolderAndHideMembersIfNecessary(SubMonitor monitor, Set<IPath> alwaysAllowedFolders,
boolean foundWorkspaceRoot, Set<IPath> visiblePaths, int maxDepth, IContainer container)
throws CoreException {
monitor.subTask(container.getFullPath().toString());

// ensure the folder is up to date
container.refreshLocal(DEPTH_ONE, monitor.slice(1));

// check all its children
var members = container.members(INCLUDE_HIDDEN);
for (IResource resource : members) {
// we only hide folders, i.e. all files contained in the project remain visible
if (resource.getType() != IResource.FOLDER) {
continue;
}

var path = resource.getProjectRelativePath();
if (findPathOrAnyParentInSet(path, alwaysAllowedFolders)) {
// never hide those in the always allowed folders
resource.setHidden(false);

// continue with next member, there is no need to go any deeper here
continue;
}

// we need to check three things
// 1. if workspace root '.' is listed, everything should be visible by default
// 2. if a parent is in visiblePaths it should be visible
// 3. if a sub-sub-sub directory is in importRoots then it should be visible as well
var visible = foundWorkspaceRoot || visiblePaths.contains(resource.getProjectRelativePath())
|| importRoots.containsWorkspacePath(new WorkspacePath(path.toString()));
// but an explicit exclude dominates
var excluded = importRoots.isExcluded(new WorkspacePath(path.toString()));

// summarize hidden state
var isHidden = excluded || !visible;

// toggle the resource hidden state
resource.setHidden(isHidden);

// there was no update to the resource
// in for efficiency of process don't go deeper if a folder is hidden
if (isHidden) {
continue;
}

// folder is visible then check its children
if ((maxDepth - 1) > 0) {
refreshFolderAndHideMembersIfNecessary(
monitor.newChild(1),
alwaysAllowedFolders,
foundWorkspaceRoot,
visiblePaths,
maxDepth - 1,
(IContainer) resource);
}
}
}

private void removeObsoleteProjects(List<BazelProject> provisionedProjects, SubMonitor monitor)
throws CoreException {
var obsoleteProjects = new ArrayList<IProject>();
Expand Down Expand Up @@ -520,7 +553,7 @@ public IStatus runInWorkspace(IProgressMonitor monitor) throws CoreException {
var workspaceName = workspace.getName();
var workspaceRoot = workspace.getLocation();

var progress = SubMonitor.convert(monitor, format("Synchronizing workspace %s", workspaceName), 20);
var progress = SubMonitor.convert(monitor, format("Synchronizing workspace %s", workspaceName), 50);

// import preferences
importPreferences(projectView.importPreferences(), progress.split(1, SUPPRESS_NONE));
Expand All @@ -546,16 +579,18 @@ public IStatus runInWorkspace(IProgressMonitor monitor) throws CoreException {
}

// ensure Bazel symlinks are filtered
configureFiltersAndRefresh(workspaceProject, progress.split(1, SUPPRESS_NONE));
configureFilters(workspaceProject, progress.split(5, SUPPRESS_NONE));

// apply excludes
hideFoldersNotVisibleAccordingToProjectView(workspaceProject, progress.split(10, SUPPRESS_NONE));
hideFoldersNotVisibleAccordingToProjectViewAndSmartRefresh(
workspaceProject,
progress.split(10, SUPPRESS_NONE));

// detect targets
var targets = detectTargetsToMaterializeInEclipse(workspaceProject, progress.split(1, SUPPRESS_NONE));
var targets = detectTargetsToMaterializeInEclipse(workspaceProject, progress.split(5, SUPPRESS_NONE));

// ensure project exists
var targetProjects = provisionProjectsForTarget(targets, progress.split(10, SUPPRESS_NONE));
var targetProjects = provisionProjectsForTarget(targets, progress.split(20, SUPPRESS_NONE));

// remove no longer needed projects
removeObsoleteProjects(targetProjects, progress.split(1, SUPPRESS_NONE));
Expand Down
6 changes: 3 additions & 3 deletions ....core/src/com/salesforce/bazel/eclipse/core/model/discovery/BaseProvisioningStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1139,7 +1139,7 @@ protected void linkSourcesIntoProject(BazelProject project, JavaProjectInfo java
public List<BazelProject> provisionProjectsForSelectedTargets(Collection<BazelTarget> targets,
BazelWorkspace workspace, IProgressMonitor progress) throws CoreException {
try {
var monitor = SubMonitor.convert(progress, "Provisioning projects", 2);
var monitor = SubMonitor.convert(progress, "Provisioning projects", 3);

// load all packages to be provisioned
workspace.open(targets.stream().map(BazelTarget::getBazelPackage).distinct().toList());
Expand All @@ -1157,10 +1157,10 @@ public List<BazelProject> provisionProjectsForSelectedTargets(Collection<BazelTa
detectDefaultJavaToolchain(workspace);

// create projects
var result = doProvisionProjects(targets, monitor.split(1));
var result = doProvisionProjects(targets, monitor.split(1, SUPPRESS_NONE));

// after provisioning we go over the projects a second time to initialize the classpaths
doInitializeClasspaths(result, workspace, monitor.split(1));
doInitializeClasspaths(result, workspace, monitor.split(2, SUPPRESS_NONE));

// done
return result;
Expand Down
11 changes: 7 additions & 4 deletions .../salesforce/bazel/eclipse/core/model/discovery/ProjectPerPackageProvisioningStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.joining;
import static java.util.stream.Collectors.toList;
import static org.eclipse.core.runtime.SubMonitor.SUPPRESS_ALL_LABELS;

import java.util.ArrayList;
import java.util.Collection;
Expand Down Expand Up @@ -60,7 +61,7 @@ public Map<BazelProject, Collection<ClasspathEntry>> computeClasspaths(Collectio
BazelWorkspace workspace, BazelClasspathScope scope, IProgressMonitor progress) throws CoreException {
LOG.debug("Computing classpath for projects: {}", bazelProjects);
try {
var monitor = SubMonitor.convert(progress, "Computing classpaths...", 1 + bazelProjects.size());
var monitor = SubMonitor.convert(progress, "Computing Bazel project classpaths", 1 + bazelProjects.size());

List<BazelLabel> targetsToBuild = new ArrayList<>(bazelProjects.size());
Map<BazelProject, List<BazelTarget>> activeTargetsPerProject = new HashMap<>();
Expand Down Expand Up @@ -116,18 +117,18 @@ public Map<BazelProject, Collection<ClasspathEntry>> computeClasspaths(Collectio
onlyDirectDeps,
"Running build with IntelliJ aspects to collect classpath information");

monitor.subTask("Running Bazel...");
monitor.subTask("Running Bazel build with aspects");
var result = workspace.getCommandExecutor()
.runDirectlyWithWorkspaceLock(
command,
bazelProjects.stream().map(BazelProject::getProject).collect(toList()),
monitor.split(1));
monitor.split(1, SUPPRESS_ALL_LABELS));

// populate map from result
Map<BazelProject, Collection<ClasspathEntry>> classpathsByProject = new HashMap<>();
var aspectsInfo = new JavaAspectsInfo(result, workspace);
for (BazelProject bazelProject : bazelProjects) {
monitor.subTask("Analyzing: " + bazelProject);
monitor.subTask(bazelProject.getName());
monitor.checkCanceled();

// build index of classpath info
Expand Down Expand Up @@ -205,6 +206,8 @@ protected List<BazelProject> doProvisionProjects(Collection<BazelTarget> targets
var bazelPackage = entry.getKey();
var packageTargets = entry.getValue();

monitor.subTask(bazelPackage.getName());

// skip the root package (not supported)
if (bazelPackage.isRoot()) {
createBuildPathProblem(
Expand Down
Loading

0 comments on commit bcdd293

Please sign in to comment.