diff --git a/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1A/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240.json b/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1A/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240.json index ca807d00..68b0bf0b 100644 --- a/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1A/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240.json +++ b/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1A/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240.json @@ -38,6 +38,8 @@ }, "properties": { "sat:orbit_state": "ascending", + "saocom:path": 149, + "saocom:row": 447, "proj:epsg": null, "proj:shape": [ 6419, @@ -105,8 +107,8 @@ "roles": [ "metadata" ], - "href": "S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh.xml", - "filename": "Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh.xml" + "href": "data/286806-EOL1ASARSAO1A3111864/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh.xml", + "filename": "Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh.xml", }, "amplitude-hh": { "type": "image/tiff; application=geotiff", @@ -114,7 +116,7 @@ "amplitude", "data" ], - "href": "S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh", + "href": "data/286806-EOL1ASARSAO1A3111864/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh", "filename": "Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh", "sar:polarizations": [ "HH" @@ -125,8 +127,8 @@ "roles": [ "metadata" ], - "href": "S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv.xml", - "filename": "Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv.xml" + "href": "data/286806-EOL1ASARSAO1A3111864/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv.xml", + "filename": "Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv.xml", }, "amplitude-hv": { "type": "image/tiff; application=geotiff", @@ -134,7 +136,7 @@ "amplitude", "data" ], - "href": "S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv", + "href": "data/286806-EOL1ASARSAO1A3111864/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv", "filename": "Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv", "sar:polarizations": [ "HV" @@ -142,4 +144,4 @@ } }, "links": [] -} \ No newline at end of file +} diff --git a/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1C/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1C_OLF_20200304T225242.json b/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1C/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1C_OLF_20200304T225242.json index 608fa946..ef85717a 100644 --- a/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1C/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1C_OLF_20200304T225242.json +++ b/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1C/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1C_OLF_20200304T225242.json @@ -37,17 +37,9 @@ ] }, "properties": { - "datetime": "2019-11-08T09:50:27.048Z", - "created": "2020-03-04T23:12:48Z", - "platform": "saocom-1a", - "mission": "saocom-1", - "instruments": [ - "sao1a" - ], - "sensor_type": "radar", - "gsd": 10.0, - "title": "SAOCOM-1A SAO1A GEC VV/VH/HH/HV 2019-11-08 09:50:27", "sat:orbit_state": "descending", + "saocom:path": 34, + "saocom:row": 151, "proj:epsg": null, "proj:shape": [ 4463, @@ -71,6 +63,16 @@ "sar:resolution_range": 10.0, "sar:resolution_azimuth": 10.0, "processing:level": "L1C", + "datetime": "2019-11-08T09:50:27.048Z", + "created": "2020-03-04T23:12:48Z", + "platform": "saocom-1a", + "mission": "saocom-1", + "instruments": [ + "sao1a" + ], + "sensor_type": "radar", + "gsd": 10.0, + "title": "SAOCOM-1A SAO1A GEC VV/VH/HH/HV 2019-11-08 09:50:27", "providers": [ { "name": "CONAE", diff --git a/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1D/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1D_OLVF_20200819T231514.json b/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1D/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1D_OLVF_20200819T231514.json index f66d837d..2245f92c 100644 --- a/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1D/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1D_OLVF_20200819T231514.json +++ b/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1D/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1D_OLVF_20200819T231514.json @@ -37,16 +37,9 @@ ] }, "properties": { - "datetime": "2020-08-19T21:17:26.613Z", - "created": "2020-08-20T01:29:31Z", - "platform": "saocom-1a", - "mission": "saocom-1", - "instruments": [ - "sao1a" - ], - "sensor_type": "radar", - "title": "SAOCOM-1A SAO1A GTC VV/VH 2020-08-19 21:17:26", "sat:orbit_state": "ascending", + "saocom:row": 529, + "saocom:path": 137, "proj:epsg": null, "proj:shape": [ 9848, @@ -68,6 +61,15 @@ "sar:resolution_range": 50.0, "sar:resolution_azimuth": 50.0, "processing:level": "L1D", + "datetime": "2020-08-19T21:17:26.613Z", + "created": "2020-08-20T01:29:31Z", + "platform": "saocom-1a", + "mission": "saocom-1", + "instruments": [ + "sao1a" + ], + "sensor_type": "radar", + "title": "SAOCOM-1A SAO1A GTC VV/VH 2020-08-19 21:17:26", "providers": [ { "name": "CONAE", diff --git a/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1D/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1D_OLVF_20210204T093006.json b/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1D/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1D_OLVF_20210204T093006.json index 13ae5376..11a72bd9 100644 --- a/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1D/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1D_OLVF_20210204T093006.json +++ b/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1D/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1D_OLVF_20210204T093006.json @@ -37,16 +37,9 @@ ] }, "properties": { - "datetime": "2021-02-04T04:14:38.399Z", - "created": "2021-02-04T10:43:10Z", - "platform": "saocom-1a", - "mission": "saocom-1", - "instruments": [ - "sao1a" - ], - "sensor_type": "radar", - "title": "SAOCOM-1A SAO1A GTC VV/VH/HH/HV 2021-02-04 04:14:38", "sat:orbit_state": "ascending", + "saocom:row": 406, + "saocom:path": 209, "proj:epsg": null, "proj:shape": [ 4817, @@ -70,6 +63,15 @@ "sar:resolution_range": 50.0, "sar:resolution_azimuth": 50.0, "processing:level": "L1D", + "datetime": "2021-02-04T04:14:38.399Z", + "created": "2021-02-04T10:43:10Z", + "platform": "saocom-1a", + "mission": "saocom-1", + "instruments": [ + "sao1a" + ], + "sensor_type": "radar", + "title": "SAOCOM-1A SAO1A GTC VV/VH/HH/HV 2021-02-04 04:14:38", "providers": [ { "name": "CONAE", diff --git a/src/Stars.Data/Model/Metadata/Saocom1/Saocom1MetadataExtractor.cs b/src/Stars.Data/Model/Metadata/Saocom1/Saocom1MetadataExtractor.cs index 2238a087..a028912e 100644 --- a/src/Stars.Data/Model/Metadata/Saocom1/Saocom1MetadataExtractor.cs +++ b/src/Stars.Data/Model/Metadata/Saocom1/Saocom1MetadataExtractor.cs @@ -35,6 +35,7 @@ public class Saocom1MetadataExtractor : MetadataExtraction { public static XmlSerializer metadataSerializer = new XmlSerializer(typeof(SAOCOM_XMLProduct)); public static XmlSerializer manifestSerializer = new XmlSerializer(typeof(XEMT)); + public static XmlSerializer parametersSerializer = new XmlSerializer(typeof(ParameterFile)); private readonly IFileSystem _fileSystem; private readonly CarrierManager _carrierManager; @@ -84,10 +85,14 @@ protected override async Task ExtractMetadata(IItem item, string suffi if (zipAsset != null) { ZipArchiveAsset zipArchiveAsset = new ZipArchiveAsset(zipAsset, logger, resourceServiceProvider, _fileSystem); + zipArchiveAsset.IsInternalArchive = true; zipArchiveAsset.UseParentAssetBaseDir = true; - var tmpDestination = LocalFileDestination.Create(_fileSystem.DirectoryInfo.FromDirectoryName(Path.GetTempPath()), item); - extractedAssets = await zipArchiveAsset.ExtractToDestinationAsync(tmpDestination, _carrierManager, System.Threading.CancellationToken.None); + string dirName = Path.GetDirectoryName(zipArchiveAsset.Uri.AbsolutePath) ?? Path.GetTempPath(); + IResource route = item; + IDestination destination = LocalFileDestination.Create(_fileSystem.DirectoryInfo.FromDirectoryName(dirName), item, true); + + extractedAssets = await zipArchiveAsset.ExtractToDestinationAsync(destination, _carrierManager, System.Threading.CancellationToken.None); } if (extractedAssets != null && item is IAssetsContainer) @@ -129,6 +134,7 @@ internal virtual StacItem CreateStacItem(SAOCOM_XMLProduct metadata, XEMT manife Dictionary properties = new Dictionary(); StacItem stacItem = new StacItem(ReadFilename(item), GetGeometry(item, kml, metadata), properties); AddSatStacExtension(metadata, stacItem); + AddOrbitInformation(metadata, manifest, stacItem, item); AddProjStacExtension(metadata, stacItem); AddViewStacExtension(metadata, manifest, stacItem); AddSarStacExtension(metadata, stacItem, item); @@ -253,6 +259,37 @@ private void AddSatStacExtension(SAOCOM_XMLProduct metadata, StacItem stacItem) sat.AbsoluteOrbit = absOrbit; } + private void AddOrbitInformation(SAOCOM_XMLProduct metadata, XEMT manifest, StacItem stacItem, IItem item) + { + if (manifest != null && manifest.Product != null && manifest.Product.Features != null && manifest.Product.Features.GeographicAttributes != null && manifest.Product.Features.GeographicAttributes.PathRow != null) + { + stacItem.Properties["saocom:path"] = manifest.Product.Features.GeographicAttributes.PathRow.Path; + stacItem.Properties["saocom:row"] = manifest.Product.Features.GeographicAttributes.PathRow.Row; + } + else + { + IAsset parametersAsset = GetParametersAsset(item); + if (parametersAsset != null) + { + ParameterFile parameters = null; + using (var stream = resourceServiceProvider.GetAssetStreamAsync(parametersAsset, System.Threading.CancellationToken.None).Result) + { + var reader = XmlReader.Create(stream); + logger.LogDebug("Deserializing metadata file {0}", parametersAsset.Uri); + parameters = (ParameterFile)parametersSerializer.Deserialize(reader); + } + if (parameters.Inputs != null && parameters.Inputs.Parameters != null) + { + foreach (Parameter p in parameters.Inputs.Parameters) + { + if (p.Name == "Path" && Int32.TryParse(p.Value, out int path))stacItem.Properties["saocom:path"] = path; + if (p.Name == "Row" && Int32.TryParse(p.Value, out int row)) stacItem.Properties["saocom:row"] = row; + } + } + } + } + } + private string GetProductType(SAOCOM_XMLProduct metadata) { string fileName = metadata.Channel[0].RasterInfo.FileName; @@ -634,8 +671,7 @@ private double GetGroundSampleDistance(SAOCOM_XMLProduct metadata, XEMT manifest protected virtual IAsset GetMetadataAsset(IItem item) { - IAsset metadataAsset = null; - metadataAsset = FindFirstAssetFromFileNameRegex(item, @"(slc|di|gec|gtc)-.*\.xml"); + IAsset metadataAsset = FindFirstAssetFromFileNameRegex(item, @"(slc|di|gec|gtc)-.*\.xml"); return metadataAsset; } @@ -643,12 +679,18 @@ protected virtual IAsset GetMetadataAsset(IItem item) protected virtual IAsset GetManifestAsset(IItem item) { - IAsset manifestAsset = null; - manifestAsset = FindFirstAssetFromFileNameRegex(item, @".*\.xemt"); + IAsset manifestAsset = FindFirstAssetFromFileNameRegex(item, @".*\.xemt"); return manifestAsset; } + protected virtual IAsset GetParametersAsset(IItem item) + { + IAsset parametersAsset = FindFirstAssetFromFileNameRegex(item, @"parameterFile2\.xml"); + return parametersAsset; + } + + protected virtual IAsset GetZipAsset(IItem item) { IAsset zipAsset = null; diff --git a/src/Stars.Data/Model/Metadata/Saocom1/Schema.cs b/src/Stars.Data/Model/Metadata/Saocom1/Schema.cs index 63bc4293..a377dadd 100644 --- a/src/Stars.Data/Model/Metadata/Saocom1/Schema.cs +++ b/src/Stars.Data/Model/Metadata/Saocom1/Schema.cs @@ -360,7 +360,9 @@ public class SAOCOM_XMLProduct public string At { get; set; } } - // XEMT files (incomplete classes) + // ========================================================================= + // .xemt files (incomplete classes) + // ========================================================================= [XmlRoot(ElementName = "xemt", Namespace = "http://www.conae.gov.ar/CUSS/XEMT")] public class XEMT @@ -410,6 +412,8 @@ public class Features public Production Production { get; set; } [XmlElement(ElementName = "acquisition")] public Acquisition Acquisition { get; set; } + [XmlElement(ElementName = "geographicAttributes")] + public GeographicAttributes GeographicAttributes { get; set; } } @@ -456,5 +460,59 @@ public class AcquisitionParameters public string SideLooking { get; set; } } + [XmlRoot(ElementName = "geographicAttributes")] + public class GeographicAttributes + { + [XmlElement(ElementName = "pathRow")] + public PathRow PathRow { get; set; } + } + + [XmlRoot(ElementName = "pathRow")] + public class PathRow + { + [XmlElement(ElementName = "Path")] + public int Path { get; set; } + [XmlElement(ElementName = "Row")] + public int Row { get; set; } + } + + // ========================================================================= + // parameters2.xml + // ========================================================================= + + [XmlRoot(ElementName = "parameters", Namespace = "http://www.conae.gov.ar/CGSS/XPNet")] + public class ParameterFile + { + [XmlElement(ElementName = "inputs")] + public Inputs Inputs { get; set; } + [XmlElement(ElementName = "outputs")] + public Outputs Outputs { get; set; } + } + + [XmlRoot(ElementName = "inputs")] + public class Inputs + { + [XmlElement(ElementName = "parameter")] + public List Parameters { get; set; } + } + + [XmlRoot(ElementName = "outputs")] + public class Outputs + { + [XmlElement(ElementName = "parameter")] + public List Parameters { get; set; } + } + + [XmlRoot(ElementName = "parameter")] + public class Parameter + { + [XmlElement(ElementName = "name")] + public string Name { get; set; } + [XmlElement(ElementName = "type")] + public string Type { get; set; } + [XmlElement(ElementName = "value")] + public string Value { get; set; } + } + } \ No newline at end of file diff --git a/src/Stars.Services/Processing/ZipArchiveAsset.cs b/src/Stars.Services/Processing/ZipArchiveAsset.cs index 6d47ab27..baa78215 100644 --- a/src/Stars.Services/Processing/ZipArchiveAsset.cs +++ b/src/Stars.Services/Processing/ZipArchiveAsset.cs @@ -98,6 +98,9 @@ public bool UseParentAssetBaseDir } } + public bool IsInternalArchive { get; set; } + + // Directory where ZIP file is located // (value is set automatically if UseParentAssetBaseDir is set to true) public string ParentAssetBaseDir { get; set; } @@ -119,7 +122,7 @@ public async override Task ExtractToDestinationAsync(IDestinat { Dictionary assetsExtracted = new Dictionary(); zipFile = Ionic.Zip.ZipFile.Read(await GetZipStreamAsync(asset, carrierManager, ct)); - string subFolder = AutodetectSubfolder(); + string subFolder = IsInternalArchive ? String.Empty : AutodetectSubfolder(); foreach (var archiveAsset in Assets) { diff --git a/src/Stars.Services/Supplier/Destination/LocalFileDestination.cs b/src/Stars.Services/Supplier/Destination/LocalFileDestination.cs index dc324398..b5f63daf 100644 --- a/src/Stars.Services/Supplier/Destination/LocalFileDestination.cs +++ b/src/Stars.Services/Supplier/Destination/LocalFileDestination.cs @@ -18,7 +18,7 @@ public class LocalFileDestination : IDestination private LocalFileDestination(IFileInfo file, IResource resource) { - this.file = file; + this.file = file; // note: file can be the directory containing the zip file in case of nested zips this.resource = resource; } @@ -26,14 +26,19 @@ private LocalFileDestination(IFileInfo file, IResource resource) public bool Exists => file.Exists; - public static LocalFileDestination Create(IDirectoryInfo directory, IResource route) + // Field 'file' can refer to a directory, in that case base directory is that directory, + // otherwise the parent directory of 'file' + public string BaseDirectory => Directory.Exists(file.FullName) ? file.FullName : file.Directory.FullName; + + public static LocalFileDestination Create(IDirectoryInfo directory, IResource route, bool ignoreEmptyFilename = false) { var filename = Path.GetFileName(route.Uri.ToString()); if (route.ContentDisposition != null && !string.IsNullOrEmpty(route.ContentDisposition.FileName)) filename = route.ContentDisposition.FileName; - if (string.IsNullOrEmpty(filename)) + if (string.IsNullOrEmpty(filename) && !ignoreEmptyFilename) filename = Guid.NewGuid().ToString("N") + "/"; - return new LocalFileDestination(directory.FileSystem.FileInfo.FromFileName(Path.Combine(directory.FullName, filename)), route); + string path = filename == null ? directory.FullName : Path.Combine(directory.FullName, filename); + return new LocalFileDestination(directory.FileSystem.FileInfo.FromFileName(path), route); } public void PrepareDestination() @@ -45,7 +50,7 @@ public void PrepareDestination() public IDestination To(IResource subroute, string relPathFix = null) { // we first integrate the relPath - string relPath = relPathFix ?? ""; + string relPath = relPathFix ?? String.Empty; // we identify the filename string filename = Path.GetFileName(subroute.Uri.IsAbsoluteUri ? subroute.Uri.LocalPath : subroute.Uri.ToString()); @@ -90,7 +95,7 @@ public IDestination To(IResource subroute, string relPathFix = null) } if (relPath.StartsWith("/")) relPath = relPath.Substring(1); if (filename.StartsWith("/")) filename = filename.Substring(1); - var newFilePath = Path.Combine(file.Directory.FullName, relPath, filename); + var newFilePath = Path.Combine(BaseDirectory, relPath, filename); return new LocalFileDestination(file.FileSystem.FileInfo.FromFileName(newFilePath), subroute); }