Partial parallelization

GameRealisticMap.Arma3.CommandLine/MapWorkspace.cs

@@ -53,6 +53,7 @@ public static async Task<MapWorkspace> Create(Arma3MapConfig a3config, string se
         public void Dispose()
         {
             Progress.Dispose();
+            Console.WriteLine($"Done in {Progress.Root.ElapsedMilliseconds} msec");
         }
     }
 }

GameRealisticMap.Arma3.Test/ContextMock.cs

@@ -34,5 +34,10 @@ public void Add<T>(T value) where T : class
         {
             Add(typeof(T), value);
         }
+
+        public Task<T> GetDataAsync<T>(IProgressScope? parentScope = null) where T : class
+        {
+            return Task.FromResult(GetData<T>(parentScope));
+        }
     }
 }

GameRealisticMap.Arma3/Arma3DemoMapGenerator.cs

@@ -13,7 +13,6 @@
 using GameRealisticMap.ManMade.Places;
 using GameRealisticMap.ManMade.Roads;
 using GameRealisticMap.Osm;
-using GameRealisticMap.Reporting;
 using HugeImages.Storage;
 using Pmad.ProgressTracking;
 
@@ -129,9 +128,9 @@ private void BuildingAdjust(ElevationGrid grid, List<City> places, List<Editable
         }
 
 
-        protected override IEnumerable<EditableWrpObject> GetObjects(IProgressScope progress, IArma3MapConfig config, IContext context, Arma3LayerGeneratorCatalog generators, ElevationGrid grid)
+        protected override async Task<IEnumerable<EditableWrpObject>> GetObjects(IProgressScope progress, IArma3MapConfig config, IContext context, Arma3LayerGeneratorCatalog generators, ElevationGrid grid)
         {
-            return base.GetObjects(progress, config, context, generators, grid)
+            return (await base.GetObjects(progress, config, context, generators, grid))
                 .Concat(context.GetData<List<EditableWrpObject>>());
         }
 

GameRealisticMap.Arma3/Arma3MapGenerator.cs

@@ -1,4 +1,5 @@
-using System.Runtime.Versioning;
+using System.Collections.Concurrent;
+using System.Runtime.Versioning;
 using BIS.WRP;
 using GameRealisticMap.Arma3.Assets;
 using GameRealisticMap.Arma3.GameEngine;
@@ -94,7 +95,7 @@ protected virtual BuildContext CreateBuildContext(IProgressScope progress, Arma3
 
             // Generate content
             var context = CreateBuildContext(progress, a3config, osmSource);
-            var results = GenerateWrp(progress, a3config, context, a3config.TerrainArea, generators);
+            var results = await GenerateWrp(progress, a3config, context, a3config.TerrainArea, generators);
             context.DisposeHugeImages();
             if (progress.CancellationToken.IsCancellationRequested)
             {
@@ -116,7 +117,7 @@ protected virtual async Task<IOsmDataSource> LoadOsmData(IProgressScope progress
             return await loader.Load(a3config.TerrainArea);
         }
 
-        public WrpAndContextResults? GenerateWrp(IProgressScope progress, Arma3MapConfig config, IContext context, ITerrainArea area, Arma3LayerGeneratorCatalog generators)
+        public async ValueTask<WrpAndContextResults?> GenerateWrp(IProgressScope progress, Arma3MapConfig config, IContext context, ITerrainArea area, Arma3LayerGeneratorCatalog generators)
         {
             // Game config
             new GameConfigGenerator(assets, projectDrive).Generate(config, context, area);
@@ -133,7 +134,8 @@ protected virtual async Task<IOsmDataSource> LoadOsmData(IProgressScope progress
             // Imagery
             var imageryCompiler = new ImageryCompiler(assets.Materials, progress, projectDrive);
 
-            var tiles = imageryCompiler.Compile(config, CreateImagerySource(progress, config, context));
+            var gridTask = context.GetDataAsync<ElevationData>();
+            var tiles = await imageryCompiler.Compile(config, CreateImagerySource(progress, config, context));
             if (progress.CancellationToken.IsCancellationRequested)
             {
                 return null;
@@ -142,11 +144,10 @@ protected virtual async Task<IOsmDataSource> LoadOsmData(IProgressScope progress
             // Objects + WRP
             var wrpBuilder = new WrpCompiler(progress, projectDrive);
 
-            var grid = context.GetData<ElevationData>().Elevation;
-
+            var grid = (await gridTask).Elevation;
             var size = area.SizeInMeters;
 
-            var objects = GetObjects(progress, config, context, generators, grid)
+            var objects = (await GetObjects(progress, config, context, generators, grid))
                 .Where(o => IsStrictlyInside(o, size));
 
             wrpBuilder.Write(config, grid, tiles, objects);
@@ -161,27 +162,24 @@ protected virtual IImagerySource CreateImagerySource(IProgressScope progress, Ar
             return new ImagerySource(assets.Materials, progress, projectDrive, config, context);
         }
 
-        protected virtual IEnumerable<EditableWrpObject> GetObjects(IProgressScope progress, IArma3MapConfig config, IContext context, Arma3LayerGeneratorCatalog generators, ElevationGrid grid)
+        protected virtual async Task<IEnumerable<EditableWrpObject>> GetObjects(IProgressScope progress, IArma3MapConfig config, IContext context, Arma3LayerGeneratorCatalog generators, ElevationGrid grid)
         {
-            return GenerateObjects(progress, config, context, generators).Select(o => o.ToWrpObject(grid));
+            return (await GenerateObjects(progress, config, context, generators))
+                .SelectMany(o => o)
+                .Select(o => o.ToWrpObject(grid));
         }
 
-        private IEnumerable<TerrainBuilderObject> GenerateObjects(IProgressScope progress, IArma3MapConfig config, IContext context, Arma3LayerGeneratorCatalog generators)
+        private async ValueTask<IEnumerable<IEnumerable<TerrainBuilderObject>>> GenerateObjects(IProgressScope progress, IArma3MapConfig config, IContext context, Arma3LayerGeneratorCatalog generators)
         {
+            var result = new ConcurrentQueue<IEnumerable<TerrainBuilderObject>>();
             using (var scope = progress.CreateScope("Objects", generators.Generators.Count))
             {
-                foreach (var tb in generators.Generators)
+                await Parallel.ForEachAsync(generators.Generators, new ParallelOptions() { CancellationToken = progress.CancellationToken, MaxDegreeOfParallelism = 4 }, async (tb, _) =>
                 {
-                    if (progress.CancellationToken.IsCancellationRequested)
-                    {
-                        break;
-                    }
-                    foreach (var obj in tb.Generate(config, context, scope))
-                    {
-                        yield return obj;
-                    }
-                }
+                    result.Enqueue(await tb.Generate(config, context, scope));
+                });
             }
+            return result;
         }
 
         private bool IsStrictlyInside(EditableWrpObject o, float size)

GameRealisticMap.Arma3/Arma3TerrainBuilderGenerator.cs

@@ -11,7 +11,6 @@
 using GameRealisticMap.ManMade.Roads;
 using GameRealisticMap.Osm;
 using GameRealisticMap.Preview;
-using GameRealisticMap.Reporting;
 using HugeImages;
 using HugeImages.Processing;
 using HugeImages.Storage;
@@ -66,7 +65,7 @@ private BuildContext CreateBuildContext(IProgressScope progress, Arma3MapConfig
             }
 
 #if DEBUG
-            PreviewRender.RenderHtml(context, "debug.html").Wait();
+            await PreviewRender.RenderHtml(context, "debug.html");
 #endif
 
             // Convert PAA
@@ -130,7 +129,7 @@ private async Task ExportObjects(IProgressScope progress, Arma3MapConfig config,
             foreach (var tb in generators.Generators)
             {
                 var name = GetLayerName(tb);
-                var entries = tb.Generate(config, context, progress).ToList();
+                var entries = (await tb.Generate(config, context, progress)).ToList();
                 foreach (var entry in entries)
                 {
                     usedModels.Add(entry.Model);
@@ -187,12 +186,12 @@ private async Task<List<ImageryPart>> ExportImagery(IProgressScope progress, Arm
             var parts = GenerateParts(config);
 
             var source = new ImagerySource(assets.Materials, progress, projectDrive, config, context);
-            using (var idMap = source.CreateIdMap())
+            using (var idMap = await source.CreateIdMap())
             {
                 await WriteImage(progress, idMap, Path.Combine(targetDirectory, "idmap.png"), parts).ConfigureAwait(false);
             }
-            ImageryCompiler.CreateConfigCppImages(projectDrive, config, source);
-            using (var satMap = source.CreateSatMap())
+            await ImageryCompiler.CreateConfigCppImages(projectDrive, config, source);
+            using (var satMap = await source.CreateSatMap())
             {
                 await WriteImage(progress, satMap, Path.Combine(targetDirectory, "satmap.png"), parts).ConfigureAwait(false);
             }
@@ -413,7 +412,7 @@ public async Task GenerateOnlyOneLayer(IProgressScope progress, Arma3MapConfig a
                 throw new ApplicationException($"Layer '{layerName}' does not exists.");
             }
 
-            var result = generator.Generate(a3config, context, progress);
+            var result = await generator.Generate(a3config, context, progress);
             await WriteLayers(targetDirectory, GetLayerName(generator), result.ToList());
         }
 

GameRealisticMap.Arma3/Demo/Arma3GdtDemoImagerySource.cs

@@ -29,7 +29,7 @@ public Arma3GdtDemoImagerySource(IProgressScope progress, Arma3MapConfig config,
             this.defaultMaterial = materials.GetMaterialByUsage(TerrainMaterialUsage.Default);
         }
 
-        public HugeImage<Rgba32> CreateIdMap()
+        public async Task<HugeImage<Rgba32>> CreateIdMap()
         {
             Rgba32 defaultColor = default;
             defaultMaterial.Id.ToRgba32(ref defaultColor);
@@ -38,7 +38,7 @@ public HugeImage<Rgba32> CreateIdMap()
             var x = 0;
             var y = 0;
             var opt = new DrawingOptions() { GraphicsOptions = new GraphicsOptions() { Antialias = false } };
-            image.MutateAllAsync(p =>
+            await image.MutateAllAsync(p =>
             {
                 p.Fill(defaultColor);
                 foreach (var def in materials.Definitions)
@@ -51,24 +51,24 @@ public HugeImage<Rgba32> CreateIdMap()
                         y += SquareSizeInPixels;
                     }
                 }
-            }).GetAwaiter().GetResult();
+            });
             return image;
         }
 
-        public Image CreatePictureMap()
+        public Task<Image> CreatePictureMap()
         {
             var img = new Image<Rgba32>(256, 256, new Rgba32(255,255,255,255));
-            return img;
+            return Task.FromResult((Image)img);
         }
 
-        public HugeImage<Rgba32> CreateSatMap()
+        public async Task<HugeImage<Rgba32>> CreateSatMap()
         {
             var size = config.GetSatMapSize();
             var image = new HugeImage<Rgba32>(context.HugeImageStorage, GetType().Name, new Size(size.Width, size.Height));
             var x = 0;
             var y = 0;
             var defaultBrush = new ImageBrush(Image.Load(defaultMaterial.FakeSatPngImage));
-            image.MutateAllAsync(p =>
+            await image.MutateAllAsync(p =>
             {
                 p.Fill(defaultBrush);
                 foreach (var def in materials.Definitions)
@@ -82,7 +82,7 @@ public HugeImage<Rgba32> CreateSatMap()
                         y += SquareSizeInPixels;
                     }
                 }
-            }).GetAwaiter().GetResult();
+            });
             return image;
 
         }

GameRealisticMap.Arma3/Demo/Arma3GdtDemoMapGenerator.cs

@@ -9,7 +9,6 @@
 using GameRealisticMap.ManMade.Roads;
 using GameRealisticMap.Nature.Weather;
 using GameRealisticMap.Osm;
-using GameRealisticMap.Reporting;
 using HugeImages.Storage;
 using Pmad.ProgressTracking;
 
@@ -68,9 +67,9 @@ protected override BuildContext CreateBuildContext(IProgressScope progress, Arma
             return context;
         }
 
-        protected override IEnumerable<EditableWrpObject> GetObjects(IProgressScope progress, IArma3MapConfig config, IContext context, Arma3LayerGeneratorCatalog generators, ElevationGrid grid)
+        protected override Task<IEnumerable<EditableWrpObject>> GetObjects(IProgressScope progress, IArma3MapConfig config, IContext context, Arma3LayerGeneratorCatalog generators, ElevationGrid grid)
         {
-            return new List<EditableWrpObject>();
+            return Task.FromResult((IEnumerable<EditableWrpObject>) new List<EditableWrpObject>());
         }
 
         protected override IImagerySource CreateImagerySource(IProgressScope progress, Arma3MapConfig config, IContext context)

GameRealisticMap.Arma3/GameEngine/IImagerySource.cs

@@ -6,11 +6,11 @@ namespace GameRealisticMap.Arma3.GameEngine
 {
     public interface IImagerySource
     {
-        HugeImage<Rgba32> CreateIdMap();
+        Task<HugeImage<Rgba32>> CreateIdMap();
 
-        Image CreatePictureMap();
+        Task<Image> CreatePictureMap();
 
-        HugeImage<Rgba32> CreateSatMap();
+        Task<HugeImage<Rgba32>> CreateSatMap();
 
         Image CreateSatOut();
     }

GameRealisticMap.Arma3/GameEngine/ImageryCompiler.cs

@@ -47,37 +47,46 @@ public ImageryCompiler(TerrainMaterialLibrary materialLibrary, IProgressScope pr
             this.gameFileSystemWriter = gameFileSystemWriter;
         }
 
-        public ImageryTiler Compile(IArma3MapConfig config, IImagerySource source)
+        public async ValueTask<ImageryTiler> Compile(IArma3MapConfig config, IImagerySource source)
         {
             gameFileSystemWriter.CreateDirectory($"{config.PboPrefix}\\data\\layers");
 
             TerrainMaterialHelper.UnpackEmbeddedFiles(materialLibrary, progress, gameFileSystemWriter, config);
 
             var tiler = new ImageryTiler(config);
 
-            using (var idMap = source.CreateIdMap())
+            var idTask = Task.Run(async () =>
             {
-                // idMap.SaveAsPng("idmap.png");
-                GenerateIdMapTilesAndRvMat(config, idMap, tiler);
-            }
+                using (var idMap = await source.CreateIdMap())
+                {
+                    // idMap.SaveAsPng("idmap.png");
+                    GenerateIdMapTilesAndRvMat(config, idMap, tiler);
+                }
+            });
 
-            CreateConfigCppImages(gameFileSystemWriter, config, source);
+            await CreateConfigCppImages(gameFileSystemWriter, config, source);
 
-            using (var satMap = source.CreateSatMap())
+            var satTask = Task.Run(async () =>
             {
-                // satMap.SaveAsPng("satmap.png");
-                GenerateSatMapTiles(config, satMap, tiler);
-            }
+                using (var satMap = await source.CreateSatMap())
+                {
+                    // satMap.SaveAsPng("satmap.png");
+                    GenerateSatMapTiles(config, satMap, tiler);
+                }
+            });
+
+            await idTask;
+            await satTask;
 
             return tiler;
         }
 
-        public static void CreateConfigCppImages(IGameFileSystemWriter gameFileSystemWriter, IArma3MapConfig config, IImagerySource source)
+        public static async ValueTask CreateConfigCppImages(IGameFileSystemWriter gameFileSystemWriter, IArma3MapConfig config, IImagerySource source)
         {
             var picturemapFile = $"{config.PboPrefix}\\data\\picturemap_ca.png";
             //if (!gameFileSystemWriter.FileExists(picturemapFile))
             {
-                using (var satMapOut = source.CreatePictureMap())
+                using (var satMapOut = await source.CreatePictureMap())
                 {
                     gameFileSystemWriter.WritePngImage(picturemapFile, satMapOut);
                 }

GameRealisticMap.Arma3/ITerrainBuilderLayerGenerator.cs

@@ -5,6 +5,6 @@ namespace GameRealisticMap.Arma3
 {
     public interface ITerrainBuilderLayerGenerator
     {
-        IEnumerable<TerrainBuilderObject> Generate(IArma3MapConfig config, IContext context, IProgressScope scope);
+        Task<IEnumerable<TerrainBuilderObject>> Generate(IArma3MapConfig config, IContext context, IProgressScope scope);
     }
 }

GameRealisticMap.Arma3/Imagery/FakeSatRender.cs

@@ -24,11 +24,11 @@ public FakeSatRender(TerrainMaterialLibrary materialLibrary, IProgressScope prog
             this.gameFileSystem = gameFileSystem;
         }
 
-        public override HugeImage<Rgba32> Render(IArma3MapConfig config, IContext context)
+        public override async Task<HugeImage<Rgba32>> Render(IArma3MapConfig config, IContext context)
         {
-            var image = base.Render(config, context);
+            var image = await base.Render(config, context);
             using var step = scope.CreateSingle("FakeSatBlur");
-            image.MutateAllAsync(d => d.GaussianBlur(1.5f)).GetAwaiter().GetResult();
+            await image.MutateAllAsync(d => d.GaussianBlur(1.5f));
             return image;
         }
 

GameRealisticMap.Arma3/Imagery/IdMapRenderBase.cs

@@ -34,13 +34,13 @@ public IdMapRenderBase(TerrainMaterialLibrary materialLibrary, IProgressScope pr
             drawingOptions = new DrawingOptions();
         }
 
-        public virtual HugeImage<TPixel> Render(IArma3MapConfig config, IContext context)
+        public virtual async Task<HugeImage<TPixel>> Render(IArma3MapConfig config, IContext context)
         {
             var size = GetImageSize(config);
 
             var image = new HugeImage<TPixel>(context.HugeImageStorage, GetType().Name, new Size(size.Width, size.Height));
 
-            image.MutateAllAsync(d =>
+            await image.MutateAllAsync(d =>
             {
                 d.Fill(GetBrush(materialLibrary.GetMaterialByUsage(TerrainMaterialUsage.Default)));
 
@@ -82,7 +82,7 @@ public virtual HugeImage<TPixel> Render(IArma3MapConfig config, IContext context
 
                 DrawPolygons(config, d, TerrainMaterialUsage.Asphalt, context.GetData<AsphaltData>().Polygons);
 
-            }).GetAwaiter().GetResult();
+            });
 
             return image;
         }

GameRealisticMap.Arma3/Imagery/ImagerySource.cs

@@ -25,18 +25,18 @@ public ImagerySource(TerrainMaterialLibrary materialLibrary, IProgressScope prog
             this.context = context;
         }
 
-        public HugeImage<Rgba32> CreateIdMap()
+        public async Task<HugeImage<Rgba32>> CreateIdMap()
         {
             using var step = progress.CreateScope("Draw IdMap");
-            return new IdMapRender(materialLibrary, step).Render(config, context);
+            return await new IdMapRender(materialLibrary, step).Render(config, context);
         }
 
-        public Image CreatePictureMap()
+        public Task<Image> CreatePictureMap()
         {
             return satMapRender.RenderPictureMap(config, context, 2048);
         }
 
-        public HugeImage<Rgba32> CreateSatMap()
+        public Task<HugeImage<Rgba32>> CreateSatMap()
         {
             return satMapRender.Render(config, context);
         }