Speedup Ninja backend with many extract_objects or targets

mesonbuild/arglist.py

@@ -97,15 +97,32 @@ class CompilerArgs(T.MutableSequence[str]):
     def __init__(self, compiler: T.Union['Compiler', 'StaticLinker'],
                  iterable: T.Optional[T.Iterable[str]] = None):
         self.compiler = compiler
+
+        if isinstance(iterable, CompilerArgs):
+            iterable.flush_pre_post()
+            # list(iter(x)) is over two times slower than list(x), so
+            # pass the underlying list to list() directly, instead of an iterator
+            iterable = iterable._container
         self._container: T.List[str] = list(iterable) if iterable is not None else []
+
         self.pre: T.Deque[str] = collections.deque()
-        self.post: T.Deque[str] = collections.deque()
+        self.post: T.List[str] = []
+        self.needs_override_check: bool = False
 
     # Flush the saved pre and post list into the _container list
     #
     # This correctly deduplicates the entries after _can_dedup definition
     # Note: This function is designed to work without delete operations, as deletions are worsening the performance a lot.
     def flush_pre_post(self) -> None:
+        if not self.needs_override_check:
+            if self.pre:
+                self._container[0:0] = self.pre
+                self.pre.clear()
+            if self.post:
+                self._container.extend(self.post)
+                self.post.clear()
+            return
+
         new: T.List[str] = []
         pre_flush_set: T.Set[str] = set()
         post_flush: T.Deque[str] = collections.deque()
@@ -127,19 +144,18 @@ def flush_pre_post(self) -> None:
 
         #pre and post will overwrite every element that is in the container
         #only copy over args that are in _container but not in the post flush or pre flush set
-        if pre_flush_set or post_flush_set:
-            for a in self._container:
-                if a not in post_flush_set and a not in pre_flush_set:
-                    new.append(a)
-        else:
-            new.extend(self._container)
+        for a in self._container:
+            if a not in post_flush_set and a not in pre_flush_set:
+                new.append(a)
         new.extend(post_flush)
 
         self._container = new
         self.pre.clear()
         self.post.clear()
+        self.needs_override_check = False
 
     def __iter__(self) -> T.Iterator[str]:
+        # see also __init__, where this method is essentially inlined
         self.flush_pre_post()
         return iter(self._container)
 
@@ -287,6 +303,8 @@ def __iadd__(self, args: T.Iterable[str]) -> 'CompilerArgs':
                 # Argument already exists and adding a new instance is useless
                 if arg in self._container or arg in self.pre or arg in self.post:
                     continue
+            elif dedup is Dedup.OVERRIDDEN:
+                self.needs_override_check = True
             if self._should_prepend(arg):
                 tmp_pre.appendleft(arg)
             else:

mesonbuild/backend/backends.py

@@ -467,42 +467,36 @@ def relpath(todir: str, fromdir: str) -> str:
 
     def flatten_object_list(self, target: build.BuildTarget, proj_dir_to_build_root: str = ''
                             ) -> T.Tuple[T.List[str], T.List[build.BuildTargetTypes]]:
-        obj_list, deps = self._flatten_object_list(target, target.get_objects(), proj_dir_to_build_root)
-        return list(dict.fromkeys(obj_list)), deps
+        obj_list: T.List[str] = []
+        deps: T.List[build.BuildTargetTypes] = []
+        self._flatten_object_list(target, target.get_objects(), obj_list, deps)
+        obj_list = list(dict.fromkeys(obj_list))
+        if proj_dir_to_build_root:
+            obj_list = [os.path.join(proj_dir_to_build_root, x) for x in obj_list]
+        return obj_list, deps
 
-    def determine_ext_objs(self, objects: build.ExtractedObjects, proj_dir_to_build_root: str = '') -> T.List[str]:
-        obj_list, _ = self._flatten_object_list(objects.target, [objects], proj_dir_to_build_root)
+    def determine_ext_objs(self, objects: build.ExtractedObjects) -> T.List[str]:
+        obj_list: T.List[str] = []
+        deps: T.List[build.BuildTargetTypes] = []
+        self._flatten_object_list(objects.target, [objects], obj_list, deps)
         return list(dict.fromkeys(obj_list))
 
     def _flatten_object_list(self, target: build.BuildTarget,
                              objects: T.Sequence[T.Union[str, 'File', build.ExtractedObjects]],
-                             proj_dir_to_build_root: str) -> T.Tuple[T.List[str], T.List[build.BuildTargetTypes]]:
-        obj_list: T.List[str] = []
-        deps: T.List[build.BuildTargetTypes] = []
+                             obj_list: T.List[str], deps: T.List[build.BuildTargetTypes]) -> None:
         for obj in objects:
             if isinstance(obj, str):
-                o = os.path.join(proj_dir_to_build_root,
-                                 self.build_to_src, target.get_subdir(), obj)
+                o = os.path.join(self.build_to_src, target.get_subdir(), obj)
                 obj_list.append(o)
             elif isinstance(obj, mesonlib.File):
-                if obj.is_built:
-                    o = os.path.join(proj_dir_to_build_root,
-                                     obj.rel_to_builddir(self.build_to_src))
-                    obj_list.append(o)
-                else:
-                    o = os.path.join(proj_dir_to_build_root,
-                                     self.build_to_src)
-                    obj_list.append(obj.rel_to_builddir(o))
+                obj_list.append(obj.rel_to_builddir(self.build_to_src))
             elif isinstance(obj, build.ExtractedObjects):
                 if obj.recursive:
-                    objs, d = self._flatten_object_list(obj.target, obj.objlist, proj_dir_to_build_root)
-                    obj_list.extend(objs)
-                    deps.extend(d)
-                obj_list.extend(self._determine_ext_objs(obj, proj_dir_to_build_root))
+                    self._flatten_object_list(obj.target, obj.objlist, obj_list, deps)
+                self._determine_ext_objs(obj, obj_list)
                 deps.append(obj.target)
             else:
                 raise MesonException('Unknown data type in object list.')
-        return obj_list, deps
 
     @staticmethod
     def is_swift_target(target: build.BuildTarget) -> bool:
@@ -872,18 +866,15 @@ def object_filename_from_source(self, target: build.BuildTarget, source: 'FileOr
             return os.path.join(targetdir, ret)
         return ret
 
-    def _determine_ext_objs(self, extobj: 'build.ExtractedObjects', proj_dir_to_build_root: str) -> T.List[str]:
-        result: T.List[str] = []
-
+    def _determine_ext_objs(self, extobj: 'build.ExtractedObjects', result: T.List[str]) -> None:
         targetdir = self.get_target_private_dir(extobj.target)
 
         # Merge sources and generated sources
         raw_sources = list(extobj.srclist)
         for gensrc in extobj.genlist:
             for r in gensrc.get_outputs():
                 path = self.get_target_generated_dir(extobj.target, gensrc, r)
-                dirpart, fnamepart = os.path.split(path)
-                raw_sources.append(File(True, dirpart, fnamepart))
+                raw_sources.append(File.from_built_relative(path))
 
         # Filter out headers and all non-source files
         sources: T.List['FileOrString'] = []
@@ -899,11 +890,11 @@ def _determine_ext_objs(self, extobj: 'build.ExtractedObjects', proj_dir_to_buil
                 compiler = extobj.target.compilers[lang]
                 if compiler.get_argument_syntax() == 'msvc':
                     objname = self.get_msvc_pch_objname(lang, pch)
-                    result.append(os.path.join(proj_dir_to_build_root, targetdir, objname))
+                    result.append(os.path.join(targetdir, objname))
 
         # extobj could contain only objects and no sources
         if not sources:
-            return result
+            return
 
         # With unity builds, sources don't map directly to objects,
         # we only support extracting all the objects in this mode,
@@ -925,10 +916,7 @@ def _determine_ext_objs(self, extobj: 'build.ExtractedObjects', proj_dir_to_buil
 
         for osrc in sources:
             objname = self.object_filename_from_source(extobj.target, osrc, targetdir)
-            objpath = os.path.join(proj_dir_to_build_root, objname)
-            result.append(objpath)
-
-        return result
+            result.append(objname)
 
     def get_pch_include_args(self, compiler: 'Compiler', target: build.BuildTarget) -> T.List[str]:
         args: T.List[str] = []

mesonbuild/backend/ninjabackend.py

@@ -123,21 +123,19 @@ def get_rsp_threshold() -> int:
 NINJA_QUOTE_VAR_PAT = re.compile(r"[$ \n]")
 
 def ninja_quote(text: str, is_build_line: bool = False) -> str:
-    if is_build_line:
-        quote_re = NINJA_QUOTE_BUILD_PAT
-    else:
-        quote_re = NINJA_QUOTE_VAR_PAT
-    # Fast path for when no quoting is necessary
-    if not quote_re.search(text):
-        return text
     if '\n' in text:
         errmsg = f'''Ninja does not support newlines in rules. The content was:
 
 {text}
 
 Please report this error with a test case to the Meson bug tracker.'''
         raise MesonException(errmsg)
-    return quote_re.sub(r'$\g<0>', text)
+
+    quote_re = NINJA_QUOTE_BUILD_PAT if is_build_line else NINJA_QUOTE_VAR_PAT
+    if ' ' in text or '$' in text or (is_build_line and ':' in text):
+        return quote_re.sub(r'$\g<0>', text)
+
+    return text
 
 
 @dataclass
@@ -3313,7 +3311,7 @@ def get_link_whole_args(self, linker: DynamicLinker, target):
             objects_from_static_libs: T.List[ExtractedObjects] = []
             for dep in target.link_whole_targets:
                 l = dep.extract_all_objects(False)
-                objects_from_static_libs += self.determine_ext_objs(l, '')
+                objects_from_static_libs += self.determine_ext_objs(l)
                 objects_from_static_libs.extend(self.flatten_object_list(dep)[0])
 
             return objects_from_static_libs

mesonbuild/compilers/compilers.py

@@ -133,11 +133,15 @@ def is_header(fname: 'mesonlib.FileOrString') -> bool:
 def is_source_suffix(suffix: str) -> bool:
     return suffix in source_suffixes
 
+@lru_cache(maxsize=None)
+def cached_is_source_by_name(fname: str) -> bool:
+    suffix = fname.split('.')[-1].lower()
+    return is_source_suffix(suffix)
+
 def is_source(fname: 'mesonlib.FileOrString') -> bool:
     if isinstance(fname, mesonlib.File):
         fname = fname.fname
-    suffix = fname.split('.')[-1].lower()
-    return is_source_suffix(suffix)
+    return cached_is_source_by_name(fname)
 
 def is_assembly(fname: 'mesonlib.FileOrString') -> bool:
     if isinstance(fname, mesonlib.File):
@@ -152,14 +156,14 @@ def is_llvm_ir(fname: 'mesonlib.FileOrString') -> bool:
     return suffix in llvm_ir_suffixes
 
 @lru_cache(maxsize=None)
-def cached_by_name(fname: 'mesonlib.FileOrString') -> bool:
+def cached_is_object_by_name(fname: str) -> bool:
     suffix = fname.split('.')[-1]
     return suffix in obj_suffixes
 
 def is_object(fname: 'mesonlib.FileOrString') -> bool:
     if isinstance(fname, mesonlib.File):
         fname = fname.fname
-    return cached_by_name(fname)
+    return cached_is_object_by_name(fname)
 
 def is_library(fname: 'mesonlib.FileOrString') -> bool:
     if isinstance(fname, mesonlib.File):

mesonbuild/utils/universal.py

@@ -400,13 +400,15 @@ def from_source_file(source_root: str, subdir: str, fname: str) -> 'File':
         return File(False, subdir, fname)
 
     @staticmethod
+    @lru_cache(maxsize=None)
     def from_built_file(subdir: str, fname: str) -> 'File':
         return File(True, subdir, fname)
 
     @staticmethod
+    @lru_cache(maxsize=None)
     def from_built_relative(relative: str) -> 'File':
         dirpart, fnamepart = os.path.split(relative)
-        return File(True, dirpart, fnamepart)
+        return File.from_built_file(dirpart, fnamepart)
 
     @staticmethod
     def from_absolute_file(fname: str) -> 'File':