Experimental parallelization

jetelain · Aug 22, 2024 · 7df46e4 · 7df46e4
1 parent eac83b0
commit 7df46e4
Show file tree

Hide file tree

Showing 52 changed files with 337 additions and 86 deletions.
diff --git a/GameRealisticMap.Arma3.CommandLine/MapWorkspace.cs b/GameRealisticMap.Arma3.CommandLine/MapWorkspace.cs
@@ -53,6 +53,7 @@ public static async Task<MapWorkspace> Create(Arma3MapConfig a3config, string se
         public void Dispose()
         {
             Progress.Dispose();
+            Console.WriteLine($"Done in {Progress.Root.ElapsedMilliseconds} msec");
         }
     }
 }
diff --git a/GameRealisticMap.Arma3.Test/ContextMock.cs b/GameRealisticMap.Arma3.Test/ContextMock.cs
@@ -34,5 +34,10 @@ public void Add<T>(T value) where T : class
         {
             Add(typeof(T), value);
         }
+
+        public Task<T> GetDataAsync<T>(IProgressScope? parentScope = null) where T : class
+        {
+            return Task.FromResult(GetData<T>(parentScope));
+        }
     }
 }
diff --git a/GameRealisticMap.Arma3/Arma3MapGenerator.cs b/GameRealisticMap.Arma3/Arma3MapGenerator.cs
@@ -1,4 +1,5 @@
-using System.Runtime.Versioning;
+using System.Collections.Concurrent;
+using System.Runtime.Versioning;
 using BIS.WRP;
 using GameRealisticMap.Arma3.Assets;
 using GameRealisticMap.Arma3.GameEngine;
@@ -133,6 +134,7 @@ protected virtual async Task<IOsmDataSource> LoadOsmData(IProgressScope progress
             // Imagery
             var imageryCompiler = new ImageryCompiler(assets.Materials, progress, projectDrive);
 
+            var gridTask = context.GetDataAsync<ElevationData>();
             var tiles = imageryCompiler.Compile(config, CreateImagerySource(progress, config, context));
             if (progress.CancellationToken.IsCancellationRequested)
             {
@@ -142,8 +144,7 @@ protected virtual async Task<IOsmDataSource> LoadOsmData(IProgressScope progress
             // Objects + WRP
             var wrpBuilder = new WrpCompiler(progress, projectDrive);
 
-            var grid = context.GetData<ElevationData>().Elevation;
-
+            var grid = gridTask.Result.Elevation;
             var size = area.SizeInMeters;
 
             var objects = GetObjects(progress, config, context, generators, grid)
@@ -168,20 +169,16 @@ protected virtual IEnumerable<EditableWrpObject> GetObjects(IProgressScope progr
 
         private IEnumerable<TerrainBuilderObject> GenerateObjects(IProgressScope progress, IArma3MapConfig config, IContext context, Arma3LayerGeneratorCatalog generators)
         {
+            var result = new ConcurrentQueue<IEnumerable<TerrainBuilderObject>>();
+
             using (var scope = progress.CreateScope("Objects", generators.Generators.Count))
             {
-                foreach (var tb in generators.Generators)
+                Parallel.ForEachAsync(generators.Generators, new ParallelOptions() { CancellationToken = progress.CancellationToken, MaxDegreeOfParallelism = 4 }, async (tb, _) =>
                 {
-                    if (progress.CancellationToken.IsCancellationRequested)
-                    {
-                        break;
-                    }
-                    foreach (var obj in tb.Generate(config, context, scope))
-                    {
-                        yield return obj;
-                    }
-                }
+                    result.Enqueue(await tb.Generate(config, context, scope));
+                }).Wait();
             }
+            return result.SelectMany(o => o);
         }
 
         private bool IsStrictlyInside(EditableWrpObject o, float size)

diff --git a/GameRealisticMap.Arma3/Arma3TerrainBuilderGenerator.cs b/GameRealisticMap.Arma3/Arma3TerrainBuilderGenerator.cs
@@ -130,7 +130,7 @@ private async Task ExportObjects(IProgressScope progress, Arma3MapConfig config,
             foreach (var tb in generators.Generators)
             {
                 var name = GetLayerName(tb);
-                var entries = tb.Generate(config, context, progress).ToList();
+                var entries = (await tb.Generate(config, context, progress)).ToList();
                 foreach (var entry in entries)
                 {
                     usedModels.Add(entry.Model);
@@ -413,7 +413,7 @@ public async Task GenerateOnlyOneLayer(IProgressScope progress, Arma3MapConfig a
                 throw new ApplicationException($"Layer '{layerName}' does not exists.");
             }
 
-            var result = generator.Generate(a3config, context, progress);
+            var result = await generator.Generate(a3config, context, progress);
             await WriteLayers(targetDirectory, GetLayerName(generator), result.ToList());
         }
 

diff --git a/GameRealisticMap.Arma3/GameEngine/ImageryCompiler.cs b/GameRealisticMap.Arma3/GameEngine/ImageryCompiler.cs
@@ -55,19 +55,28 @@ public ImageryTiler Compile(IArma3MapConfig config, IImagerySource source)
 
             var tiler = new ImageryTiler(config);
 
-            using (var idMap = source.CreateIdMap())
+            var idTask = Task.Run(() =>
             {
-                // idMap.SaveAsPng("idmap.png");
-                GenerateIdMapTilesAndRvMat(config, idMap, tiler);
-            }
+                using (var idMap = source.CreateIdMap())
+                {
+                    // idMap.SaveAsPng("idmap.png");
+                    GenerateIdMapTilesAndRvMat(config, idMap, tiler);
+                }
+            });
 
             CreateConfigCppImages(gameFileSystemWriter, config, source);
 
-            using (var satMap = source.CreateSatMap())
+            var satTask = Task.Run(() =>
             {
-                // satMap.SaveAsPng("satmap.png");
-                GenerateSatMapTiles(config, satMap, tiler);
-            }
+                using (var satMap = source.CreateSatMap())
+                {
+                    // satMap.SaveAsPng("satmap.png");
+                    GenerateSatMapTiles(config, satMap, tiler);
+                }
+            });
+
+            idTask.Wait();
+            satTask.Wait();
 
             return tiler;
         }

diff --git a/GameRealisticMap.Arma3/ITerrainBuilderLayerGenerator.cs b/GameRealisticMap.Arma3/ITerrainBuilderLayerGenerator.cs
@@ -5,6 +5,6 @@ namespace GameRealisticMap.Arma3
 {
     public interface ITerrainBuilderLayerGenerator
     {
-        IEnumerable<TerrainBuilderObject> Generate(IArma3MapConfig config, IContext context, IProgressScope scope);
+        Task<IEnumerable<TerrainBuilderObject>> Generate(IArma3MapConfig config, IContext context, IProgressScope scope);
     }
 }
diff --git a/GameRealisticMap.Arma3/ManMade/BridgeGenerator.cs b/GameRealisticMap.Arma3/ManMade/BridgeGenerator.cs
@@ -18,10 +18,10 @@ public BridgeGenerator(IArma3RegionAssets assets)
             this.assets = assets;
         }
 
-        public IEnumerable<TerrainBuilderObject> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
+        public async Task<IEnumerable<TerrainBuilderObject>> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
         {
-            var grid = context.GetData<ElevationData>().Elevation;
-            var roads = context.GetData<RoadsData>().Roads.Where(r => r.SpecialSegment == WaySpecialSegment.Bridge).WithProgress(scope, "Bridges");
+            var grid = (await context.GetDataAsync<ElevationData>()).Elevation;
+            var roads = (await context.GetDataAsync<RoadsData>()).Roads.Where(r => r.SpecialSegment == WaySpecialSegment.Bridge).WithProgress(scope, "Bridges");
             var result = new List<TerrainBuilderObject>();
             foreach(var bridge in  roads)
             {

diff --git a/GameRealisticMap.Arma3/ManMade/BuildingGenerator.cs b/GameRealisticMap.Arma3/ManMade/BuildingGenerator.cs
@@ -45,11 +45,11 @@ public IEnumerable<TerrainBuilderObject> ToObjects()
             }
         }
 
-        public IEnumerable<TerrainBuilderObject> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
+        public async Task<IEnumerable<TerrainBuilderObject>> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
         {
             var result = new List<Placed>();
-            var buildings = context.GetData<BuildingsData>().Buildings;
-            var roads = context.GetData<RoadsData>().Roads;
+            var buildings = (await context.GetDataAsync<BuildingsData>()).Buildings;
+            var roads = (await context.GetDataAsync<RoadsData>()).Roads;
 
             var prevailingWind = 0f;
             if (buildings.Any(b => b.TypeId == BuildingTypeId.WindTurbine))

diff --git a/GameRealisticMap.Arma3/ManMade/Farmlands/OrchardGenerator.cs b/GameRealisticMap.Arma3/ManMade/Farmlands/OrchardGenerator.cs
@@ -21,7 +21,7 @@ public OrchardGenerator(IArma3RegionAssets assets)
             this.assets = assets;
         }
 
-        public IEnumerable<TerrainBuilderObject> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
+        public async Task<IEnumerable<TerrainBuilderObject>> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
         {
             var layer1 = new List<PlacedModel<Composition>>();
             var layer2 = new RadiusPlacedLayer<Composition>(new Vector2(config.SizeInMeters));
@@ -30,7 +30,7 @@ public IEnumerable<TerrainBuilderObject> Generate(IArma3MapConfig config, IConte
             if (lib1.Count > 0 || lib2.Count > 0)
             {
                 var small = new List<TerrainPolygon>();
-                var orchards = context.GetData<OrchardData>().Polygons;
+                var orchards = (await context.GetDataAsync<OrchardData>()).Polygons;
                 foreach (var orchard in orchards.WithProgress(scope, "Orchards"))
                 {
                     if (orchard.Area > 4000)
@@ -46,7 +46,7 @@ public IEnumerable<TerrainBuilderObject> Generate(IArma3MapConfig config, IConte
 
                 using (var subscope = scope.CreateScope("SmallOrchards"))
                 {
-                    new FillAreaBasic<Composition>(subscope, lib2).FillPolygons(layer2, small, context.GetData<ConditionEvaluator>());
+                    new FillAreaBasic<Composition>(subscope, lib2).FillPolygons(layer2, small, await context.GetDataAsync<ConditionEvaluator>());
                 }
             }
             return layer1.SelectMany(o => o.Model.ToTerrainBuilderObjects(o))

diff --git a/GameRealisticMap.Arma3/ManMade/Farmlands/VineyardsGenerator.cs b/GameRealisticMap.Arma3/ManMade/Farmlands/VineyardsGenerator.cs
@@ -17,13 +17,13 @@ public VineyardsGenerator(IArma3RegionAssets assets)
             this.assets = assets;
         }
 
-        public IEnumerable<TerrainBuilderObject> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
+        public async Task<IEnumerable<TerrainBuilderObject>> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
         {
             var layer = new List<PlacedModel<Composition>>();
             var lib = assets.GetNaturalRows(Assets.Rows.NaturalRowType.VineyardRow);
             if (lib.Count > 0)
             {
-                var vineyards = context.GetData<VineyardData>().Polygons;
+                var vineyards = (await context.GetDataAsync<VineyardData>()).Polygons;
                 foreach (var vineyard in vineyards.WithProgress(scope, "Vineyards"))
                 {
                     var rnd = RandomHelper.CreateRandom(vineyard.Centroid);

diff --git a/GameRealisticMap.Arma3/ManMade/FenceGenerator.cs b/GameRealisticMap.Arma3/ManMade/FenceGenerator.cs
@@ -17,10 +17,10 @@ public FenceGenerator(IArma3RegionAssets assets)
             this.assets = assets;
         }
 
-        public IEnumerable<TerrainBuilderObject> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
+        public async Task<IEnumerable<TerrainBuilderObject>> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
         {
-            var evaluator = context.GetData<ConditionEvaluator>();
-            var fences = context.GetData<FencesData>().Fences;
+            var evaluator = await context.GetDataAsync<ConditionEvaluator>();
+            var fences = (await context.GetDataAsync<FencesData>()).Fences;
             var layer = new List<PlacedModel<Composition>>();
             foreach (var fence in fences.WithProgress(scope, "Fences"))
             {

diff --git a/GameRealisticMap.Arma3/ManMade/OrientedObjectsGenerator.cs b/GameRealisticMap.Arma3/ManMade/OrientedObjectsGenerator.cs
@@ -16,10 +16,10 @@ public OrientedObjectsGenerator(IArma3RegionAssets assets)
             this.assets = assets;
         }
 
-        public IEnumerable<TerrainBuilderObject> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
+        public async Task<IEnumerable<TerrainBuilderObject>> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
         {
-            var evaluator = context.GetData<ConditionEvaluator>();
-            var objects = context.GetData<OrientedObjectData>().Objects;
+            var evaluator = await context.GetDataAsync<ConditionEvaluator>();
+            var objects = (await context.GetDataAsync<OrientedObjectData>()).Objects;
             var result = new List<TerrainBuilderObject>();
             foreach(var obj in objects.WithProgress(scope, "OrientedObjects"))
             {
@@ -37,7 +37,7 @@ public IEnumerable<TerrainBuilderObject> Generate(IArma3MapConfig config, IConte
             var lamps = assets.GetObjects(ObjectTypeId.StreetLamp);
             if (lamps.Count > 0)
             {
-                foreach (var obj in context.GetData<ProceduralStreetLampsData>().Objects.WithProgress(scope, "ProceduralStreetLamps"))
+                foreach (var obj in (await context.GetDataAsync<ProceduralStreetLampsData>()).Objects.WithProgress(scope, "ProceduralStreetLamps"))
                 {
                     var definition = lamps.GetRandom(obj.Point, evaluator.GetPointContext(obj.Point, obj.Road));
                     if (definition != null)

diff --git a/GameRealisticMap.Arma3/ManMade/RailwayGenerator.cs b/GameRealisticMap.Arma3/ManMade/RailwayGenerator.cs
@@ -17,13 +17,13 @@ public RailwayGenerator(IArma3RegionAssets assets)
             this.assets = assets.Railways;
         }
 
-        public IEnumerable<TerrainBuilderObject> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
+        public async Task<IEnumerable<TerrainBuilderObject>> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
         {
             if (assets == null || assets.Straights.Count == 0)
             {
                 return Enumerable.Empty<TerrainBuilderObject>();
             }
-            var railways = context.GetData<RailwaysData>().Railways;
+            var railways = (await context.GetDataAsync<RailwaysData>()).Railways;
             var layer = new List<PlacedModel<Composition>>();
             foreach (var segment in railways.WithProgress(scope, "Railways"))
             {

diff --git a/GameRealisticMap.Arma3/ManMade/SidewalksGenerator.cs b/GameRealisticMap.Arma3/ManMade/SidewalksGenerator.cs
@@ -17,14 +17,14 @@ public SidewalksGenerator(IArma3RegionAssets assets)
             this.assets = assets;
         }
 
-        public IEnumerable<TerrainBuilderObject> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
+        public async Task<IEnumerable<TerrainBuilderObject>> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
         {
             var layer = new List<PlacedModel<Composition>>();
 
             if (assets.Sidewalks.Count != 0)
             {
-                var evaluator = context.GetData<ConditionEvaluator>();
-                var paths = context.GetData<SidewalksData>().Paths;
+                var evaluator = await context.GetDataAsync<ConditionEvaluator>();
+                var paths = (await context.GetDataAsync<SidewalksData>()).Paths;
                 foreach (var path in paths.WithProgress(scope, "Sidewalks"))
                 {
                     if (path.Points.Count > 1)

diff --git a/GameRealisticMap.Arma3/Nature/GeneratorBase.cs b/GameRealisticMap.Arma3/Nature/GeneratorBase.cs
@@ -21,21 +21,21 @@ public GeneratorBase(IArma3RegionAssets assets)
 
         protected virtual bool ShouldGenerate => true;
 
-        public IEnumerable<TerrainBuilderObject> Generate(IArma3MapConfig config, IContext context, IProgressScope progress)
+        public async Task<IEnumerable<TerrainBuilderObject>> Generate(IArma3MapConfig config, IContext context, IProgressScope progress)
         {
             if (!ShouldGenerate)
             {
                 return new List<TerrainBuilderObject>(0);
             }
-            var evaluator = context.GetData<ConditionEvaluator>();
+            var evaluator = context.GetDataAsync<ConditionEvaluator>();
 
-            using var scope = progress.CreateScope(GetType().Name.Replace("Generator",""));
+            var polygons = context.GetDataAsync<TData>();
 
-            var polygons = context.GetData<TData>().Polygons;
+            using var scope = progress.CreateScope(GetType().Name.Replace("Generator", ""));
 
             var layer = new RadiusPlacedLayer<Composition>(new Vector2(config.SizeInMeters));
 
-            Generate(layer, polygons, evaluator, scope);
+            Generate(layer, (await polygons).Polygons, await evaluator, scope);
 
             return layer.SelectMany(item => item.Model.ToTerrainBuilderObjects(item));
         }

diff --git a/GameRealisticMap.Arma3/Nature/Lakes/LakeSurfaceGenerator.cs b/GameRealisticMap.Arma3/Nature/Lakes/LakeSurfaceGenerator.cs
@@ -22,9 +22,9 @@ public LakeSurfaceGenerator(IArma3RegionAssets assets)
             this.assets = assets;
         }
 
-        public IEnumerable<TerrainBuilderObject> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
+        public async Task<IEnumerable<TerrainBuilderObject>> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
         {
-            var lakes = context.GetData<ElevationWithLakesData>().Lakes;
+            var lakes = (await context.GetDataAsync<ElevationWithLakesData>()).Lakes;
             var result = new List<TerrainBuilderObject>();
             var minimalPondSize = GetMinimalPondSize();
             if (minimalPondSize != null)

diff --git a/GameRealisticMap.Arma3/Nature/Trees/TreeRowsGenerator.cs b/GameRealisticMap.Arma3/Nature/Trees/TreeRowsGenerator.cs
@@ -17,13 +17,13 @@ public TreeRowsGenerator(IArma3RegionAssets assets)
             this.assets = assets;
         }
 
-        public IEnumerable<TerrainBuilderObject> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
+        public async Task<IEnumerable<TerrainBuilderObject>> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
         {
             var layer = new List<PlacedModel<Composition>>();
             var definitions = assets.GetNaturalRows(NaturalRowType.TreeRow);
             if (definitions.Count > 0)
             {
-                var rows = context.GetData<TreeRowsData>().Rows;
+                var rows = (await context.GetDataAsync<TreeRowsData>()).Rows;
                 foreach (var row in rows.WithProgress(scope, "TreeRows"))
                 {
                     var random = RandomHelper.CreateRandom(row.FirstPoint);

diff --git a/GameRealisticMap.Arma3/Nature/Trees/TreesGenerator.cs b/GameRealisticMap.Arma3/Nature/Trees/TreesGenerator.cs
@@ -17,15 +17,15 @@ public TreesGenerator(IArma3RegionAssets assets)
             this.assets = assets;
         }
 
-        public IEnumerable<TerrainBuilderObject> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
+        public async Task<IEnumerable<TerrainBuilderObject>> Generate(IArma3MapConfig config, IContext context, IProgressScope scope)
         {
             var evaluator = context.GetData<ConditionEvaluator>();
 
             var result = new List<TerrainBuilderObject>();
             var candidates = assets.GetObjects(ObjectTypeId.Tree);
             if (candidates.Count > 0)
             {
-                var points = context.GetData<TreesData>().Points;
+                var points = (await context.GetDataAsync<TreesData>()).Points;
                 foreach (var point in points.WithProgress(scope, "Trees"))
                 {
                     var random = RandomHelper.CreateRandom(point);

diff --git a/GameRealisticMap.Test/BuildContextMock.cs b/GameRealisticMap.Test/BuildContextMock.cs
@@ -39,5 +39,10 @@ public void SetData<T>(T value)
         {
             datas[typeof(T)] = value;
         }
+
+        public Task<T> GetDataAsync<T>(IProgressScope? parentScope = null) where T : class
+        {
+            return Task.FromResult(GetData<T>(parentScope));
+        }
     }
 }