Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improved SAOCOM-1 metadata extraction #41

Merged
merged 3 commits into from
Dec 11, 2023
Merged

Improved SAOCOM-1 metadata extraction #41

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a3c1d19
Changes for correct extraction of nested SAOCOM zip files
floeschau Dec 8, 2023
4df29b2
Extract path and row values for SAOCOM-1 from parameterFile2.xml if X…
floeschau Dec 8, 2023
0347406
Adjusted expected output for one test SAOCOM-1 case (file sizes)
floeschau Dec 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions ...AOCOM-1/L1A/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@
},
"properties": {
"sat:orbit_state": "ascending",
"saocom:path": 149,
"saocom:row": 447,
"proj:epsg": null,
"proj:shape": [
6419,
Expand Down Expand Up @@ -105,16 +107,16 @@
"roles": [
"metadata"
],
"href": "S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh.xml",
"filename": "Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh.xml"
"href": "data/286806-EOL1ASARSAO1A3111864/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh.xml",
"filename": "Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh.xml",
},
"amplitude-hh": {
"type": "image/tiff; application=geotiff",
"roles": [
"amplitude",
"data"
],
"href": "S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh",
"href": "data/286806-EOL1ASARSAO1A3111864/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh",
"filename": "Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh",
"sar:polarizations": [
"HH"
Expand All @@ -125,21 +127,21 @@
"roles": [
"metadata"
],
"href": "S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv.xml",
"filename": "Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv.xml"
"href": "data/286806-EOL1ASARSAO1A3111864/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv.xml",
"filename": "Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv.xml",
},
"amplitude-hv": {
"type": "image/tiff; application=geotiff",
"roles": [
"amplitude",
"data"
],
"href": "S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv",
"href": "data/286806-EOL1ASARSAO1A3111864/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv",
"filename": "Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv",
"sar:polarizations": [
"HV"
]
}
},
"links": []
}
}
22 changes: 12 additions & 10 deletions ...AOCOM-1/L1C/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1C_OLF_20200304T225242.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,17 +37,9 @@
]
},
"properties": {
"datetime": "2019-11-08T09:50:27.048Z",
"created": "2020-03-04T23:12:48Z",
"platform": "saocom-1a",
"mission": "saocom-1",
"instruments": [
"sao1a"
],
"sensor_type": "radar",
"gsd": 10.0,
"title": "SAOCOM-1A SAO1A GEC VV/VH/HH/HV 2019-11-08 09:50:27",
"sat:orbit_state": "descending",
"saocom:path": 34,
"saocom:row": 151,
"proj:epsg": null,
"proj:shape": [
4463,
Expand All @@ -71,6 +63,16 @@
"sar:resolution_range": 10.0,
"sar:resolution_azimuth": 10.0,
"processing:level": "L1C",
"datetime": "2019-11-08T09:50:27.048Z",
"created": "2020-03-04T23:12:48Z",
"platform": "saocom-1a",
"mission": "saocom-1",
"instruments": [
"sao1a"
],
"sensor_type": "radar",
"gsd": 10.0,
"title": "SAOCOM-1A SAO1A GEC VV/VH/HH/HV 2019-11-08 09:50:27",
"providers": [
{
"name": "CONAE",
Expand Down
20 changes: 11 additions & 9 deletions ...OCOM-1/L1D/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1D_OLVF_20200819T231514.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,16 +37,9 @@
]
},
"properties": {
"datetime": "2020-08-19T21:17:26.613Z",
"created": "2020-08-20T01:29:31Z",
"platform": "saocom-1a",
"mission": "saocom-1",
"instruments": [
"sao1a"
],
"sensor_type": "radar",
"title": "SAOCOM-1A SAO1A GTC VV/VH 2020-08-19 21:17:26",
"sat:orbit_state": "ascending",
"saocom:row": 529,
"saocom:path": 137,
"proj:epsg": null,
"proj:shape": [
9848,
Expand All @@ -68,6 +61,15 @@
"sar:resolution_range": 50.0,
"sar:resolution_azimuth": 50.0,
"processing:level": "L1D",
"datetime": "2020-08-19T21:17:26.613Z",
"created": "2020-08-20T01:29:31Z",
"platform": "saocom-1a",
"mission": "saocom-1",
"instruments": [
"sao1a"
],
"sensor_type": "radar",
"title": "SAOCOM-1A SAO1A GTC VV/VH 2020-08-19 21:17:26",
"providers": [
{
"name": "CONAE",
Expand Down
20 changes: 11 additions & 9 deletions ...OCOM-1/L1D/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1D_OLVF_20210204T093006.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,16 +37,9 @@
]
},
"properties": {
"datetime": "2021-02-04T04:14:38.399Z",
"created": "2021-02-04T10:43:10Z",
"platform": "saocom-1a",
"mission": "saocom-1",
"instruments": [
"sao1a"
],
"sensor_type": "radar",
"title": "SAOCOM-1A SAO1A GTC VV/VH/HH/HV 2021-02-04 04:14:38",
"sat:orbit_state": "ascending",
"saocom:row": 406,
"saocom:path": 209,
"proj:epsg": null,
"proj:shape": [
4817,
Expand All @@ -70,6 +63,15 @@
"sar:resolution_range": 50.0,
"sar:resolution_azimuth": 50.0,
"processing:level": "L1D",
"datetime": "2021-02-04T04:14:38.399Z",
"created": "2021-02-04T10:43:10Z",
"platform": "saocom-1a",
"mission": "saocom-1",
"instruments": [
"sao1a"
],
"sensor_type": "radar",
"title": "SAOCOM-1A SAO1A GTC VV/VH/HH/HV 2021-02-04 04:14:38",
"providers": [
{
"name": "CONAE",
Expand Down
54 changes: 48 additions & 6 deletions src/Stars.Data/Model/Metadata/Saocom1/Saocom1MetadataExtractor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ public class Saocom1MetadataExtractor : MetadataExtraction
{
public static XmlSerializer metadataSerializer = new XmlSerializer(typeof(SAOCOM_XMLProduct));
public static XmlSerializer manifestSerializer = new XmlSerializer(typeof(XEMT));
public static XmlSerializer parametersSerializer = new XmlSerializer(typeof(ParameterFile));
private readonly IFileSystem _fileSystem;
private readonly CarrierManager _carrierManager;

Expand Down Expand Up @@ -84,10 +85,14 @@ protected override async Task<StacNode> ExtractMetadata(IItem item, string suffi
if (zipAsset != null)
{
ZipArchiveAsset zipArchiveAsset = new ZipArchiveAsset(zipAsset, logger, resourceServiceProvider, _fileSystem);
zipArchiveAsset.IsInternalArchive = true;
zipArchiveAsset.UseParentAssetBaseDir = true;
var tmpDestination = LocalFileDestination.Create(_fileSystem.DirectoryInfo.FromDirectoryName(Path.GetTempPath()), item);

extractedAssets = await zipArchiveAsset.ExtractToDestinationAsync(tmpDestination, _carrierManager, System.Threading.CancellationToken.None);
string dirName = Path.GetDirectoryName(zipArchiveAsset.Uri.AbsolutePath) ?? Path.GetTempPath();
IResource route = item;
IDestination destination = LocalFileDestination.Create(_fileSystem.DirectoryInfo.FromDirectoryName(dirName), item, true);

extractedAssets = await zipArchiveAsset.ExtractToDestinationAsync(destination, _carrierManager, System.Threading.CancellationToken.None);
}

if (extractedAssets != null && item is IAssetsContainer)
Expand Down Expand Up @@ -129,6 +134,7 @@ internal virtual StacItem CreateStacItem(SAOCOM_XMLProduct metadata, XEMT manife
Dictionary<string, object> properties = new Dictionary<string, object>();
StacItem stacItem = new StacItem(ReadFilename(item), GetGeometry(item, kml, metadata), properties);
AddSatStacExtension(metadata, stacItem);
AddOrbitInformation(metadata, manifest, stacItem, item);
AddProjStacExtension(metadata, stacItem);
AddViewStacExtension(metadata, manifest, stacItem);
AddSarStacExtension(metadata, stacItem, item);
Expand Down Expand Up @@ -253,6 +259,37 @@ private void AddSatStacExtension(SAOCOM_XMLProduct metadata, StacItem stacItem)
sat.AbsoluteOrbit = absOrbit;
}

private void AddOrbitInformation(SAOCOM_XMLProduct metadata, XEMT manifest, StacItem stacItem, IItem item)
{
if (manifest != null && manifest.Product != null && manifest.Product.Features != null && manifest.Product.Features.GeographicAttributes != null && manifest.Product.Features.GeographicAttributes.PathRow != null)
{
stacItem.Properties["saocom:path"] = manifest.Product.Features.GeographicAttributes.PathRow.Path;
stacItem.Properties["saocom:row"] = manifest.Product.Features.GeographicAttributes.PathRow.Row;
}
else
{
IAsset parametersAsset = GetParametersAsset(item);
if (parametersAsset != null)
{
ParameterFile parameters = null;
using (var stream = resourceServiceProvider.GetAssetStreamAsync(parametersAsset, System.Threading.CancellationToken.None).Result)
{
var reader = XmlReader.Create(stream);
logger.LogDebug("Deserializing metadata file {0}", parametersAsset.Uri);
parameters = (ParameterFile)parametersSerializer.Deserialize(reader);
}
if (parameters.Inputs != null && parameters.Inputs.Parameters != null)
{
foreach (Parameter p in parameters.Inputs.Parameters)
{
if (p.Name == "Path" && Int32.TryParse(p.Value, out int path))stacItem.Properties["saocom:path"] = path;
if (p.Name == "Row" && Int32.TryParse(p.Value, out int row)) stacItem.Properties["saocom:row"] = row;
}
}
}
}
}

private string GetProductType(SAOCOM_XMLProduct metadata)
{
string fileName = metadata.Channel[0].RasterInfo.FileName;
Expand Down Expand Up @@ -634,21 +671,26 @@ private double GetGroundSampleDistance(SAOCOM_XMLProduct metadata, XEMT manifest

protected virtual IAsset GetMetadataAsset(IItem item)
{
IAsset metadataAsset = null;
metadataAsset = FindFirstAssetFromFileNameRegex(item, @"(slc|di|gec|gtc)-.*\.xml");
IAsset metadataAsset = FindFirstAssetFromFileNameRegex(item, @"(slc|di|gec|gtc)-.*\.xml");

return metadataAsset;
}


protected virtual IAsset GetManifestAsset(IItem item)
{
IAsset manifestAsset = null;
manifestAsset = FindFirstAssetFromFileNameRegex(item, @".*\.xemt");
IAsset manifestAsset = FindFirstAssetFromFileNameRegex(item, @".*\.xemt");
return manifestAsset;
}


protected virtual IAsset GetParametersAsset(IItem item)
{
IAsset parametersAsset = FindFirstAssetFromFileNameRegex(item, @"parameterFile2\.xml");
return parametersAsset;
}


protected virtual IAsset GetZipAsset(IItem item)
{
IAsset zipAsset = null;
Expand Down
60 changes: 59 additions & 1 deletion src/Stars.Data/Model/Metadata/Saocom1/Schema.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,9 @@ public class SAOCOM_XMLProduct
public string At { get; set; }
}

// XEMT files (incomplete classes)
// =========================================================================
// .xemt files (incomplete classes)
// =========================================================================

[XmlRoot(ElementName = "xemt", Namespace = "http://www.conae.gov.ar/CUSS/XEMT")]
public class XEMT
Expand Down Expand Up @@ -410,6 +412,8 @@ public class Features
public Production Production { get; set; }
[XmlElement(ElementName = "acquisition")]
public Acquisition Acquisition { get; set; }
[XmlElement(ElementName = "geographicAttributes")]
public GeographicAttributes GeographicAttributes { get; set; }

}

Expand Down Expand Up @@ -456,5 +460,59 @@ public class AcquisitionParameters
public string SideLooking { get; set; }
}

[XmlRoot(ElementName = "geographicAttributes")]
public class GeographicAttributes
{
[XmlElement(ElementName = "pathRow")]
public PathRow PathRow { get; set; }
}

[XmlRoot(ElementName = "pathRow")]
public class PathRow
{
[XmlElement(ElementName = "Path")]
public int Path { get; set; }
[XmlElement(ElementName = "Row")]
public int Row { get; set; }
}

// =========================================================================
// parameters2.xml
// =========================================================================

[XmlRoot(ElementName = "parameters", Namespace = "http://www.conae.gov.ar/CGSS/XPNet")]
public class ParameterFile
{
[XmlElement(ElementName = "inputs")]
public Inputs Inputs { get; set; }
[XmlElement(ElementName = "outputs")]
public Outputs Outputs { get; set; }
}

[XmlRoot(ElementName = "inputs")]
public class Inputs
{
[XmlElement(ElementName = "parameter")]
public List<Parameter> Parameters { get; set; }
}

[XmlRoot(ElementName = "outputs")]
public class Outputs
{
[XmlElement(ElementName = "parameter")]
public List<Parameter> Parameters { get; set; }
}

[XmlRoot(ElementName = "parameter")]
public class Parameter
{
[XmlElement(ElementName = "name")]
public string Name { get; set; }
[XmlElement(ElementName = "type")]
public string Type { get; set; }
[XmlElement(ElementName = "value")]
public string Value { get; set; }
}


}
5 changes: 4 additions & 1 deletion src/Stars.Services/Processing/ZipArchiveAsset.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,9 @@ public bool UseParentAssetBaseDir
}
}

public bool IsInternalArchive { get; set; }


// Directory where ZIP file is located
// (value is set automatically if UseParentAssetBaseDir is set to true)
public string ParentAssetBaseDir { get; set; }
Expand All @@ -119,7 +122,7 @@ public async override Task<IAssetsContainer> ExtractToDestinationAsync(IDestinat
{
Dictionary<string, IAsset> assetsExtracted = new Dictionary<string, IAsset>();
zipFile = Ionic.Zip.ZipFile.Read(await GetZipStreamAsync(asset, carrierManager, ct));
string subFolder = AutodetectSubfolder();
string subFolder = IsInternalArchive ? String.Empty : AutodetectSubfolder();

foreach (var archiveAsset in Assets)
{
Expand Down
Loading
Loading