diff --git a/BlockBlobReader/src/blobreaderzipdeploy.json b/BlockBlobReader/src/blobreaderzipdeploy.json index 186b8503..2e11d64c 100644 --- a/BlockBlobReader/src/blobreaderzipdeploy.json +++ b/BlockBlobReader/src/blobreaderzipdeploy.json @@ -419,7 +419,7 @@ "[variables('BlobReader_resourceId')]" ], "properties": { - "packageUri": "https://appdev-cloudformation-templates.s3.amazonaws.com/AzureBlobReader/taskproducer4.1.2.zip", + "packageUri": "https://appdev-cloudformation-templates.s3.amazonaws.com/AzureBlobReader/taskproducer4.1.4.zip", "appOffline": true } } @@ -512,7 +512,7 @@ "[variables('blobreaderconsumer_resourceId')]" ], "properties": { - "packageUri": "https://appdev-cloudformation-templates.s3.amazonaws.com/AzureBlobReader/taskconsumer4.1.2.zip", + "packageUri": "https://appdev-cloudformation-templates.s3.amazonaws.com/AzureBlobReader/taskconsumer4.1.4.zip", "appOffline": true } } @@ -609,7 +609,7 @@ "[variables('DLQProcessor_resourceId')]" ], "properties": { - "packageUri": "https://appdev-cloudformation-templates.s3.amazonaws.com/AzureBlobReader/dlqprocessor4.1.2.zip", + "packageUri": "https://appdev-cloudformation-templates.s3.amazonaws.com/AzureBlobReader/dlqprocessor4.1.4.zip", "appOffline": true } } diff --git a/BlockBlobReader/src/consumer.js b/BlockBlobReader/src/consumer.js index 9aeaf4fd..d5480d52 100644 --- a/BlockBlobReader/src/consumer.js +++ b/BlockBlobReader/src/consumer.js @@ -195,8 +195,9 @@ function getRowKey(metadata) { async function setAppendBlobOffset(context, serviceBusTask, newOffset) { + let rowKey = ""; try { - let rowKey = getRowKey(serviceBusTask); + rowKey = getRowKey(serviceBusTask); // Todo: this should be atomic update if other request decreases offset it shouldn't allow context.log.verbose("Attempting to update offset row: %s from: %d to: %d", rowKey, serviceBusTask.startByte, newOffset); let entity = { @@ -208,7 +209,7 @@ async function setAppendBlobOffset(context, serviceBusTask, newOffset) { containerName: serviceBusTask.containerName, storageName: serviceBusTask.storageName } - var updateResult = await azureTableClient.updateEntity(entity, "Merge"); + let updateResult = await azureTableClient.updateEntity(entity, "Merge"); context.log.verbose("Updated offset result: %s row: %s from: %d to: %d", JSON.stringify(updateResult), rowKey, serviceBusTask.startByte, newOffset); } catch (error) { context.log.error(`Error - Failed to update OffsetMap table, error: ${JSON.stringify(error)}, rowKey: ${rowKey}, newOffset: ${newOffset}`) diff --git a/BlockBlobReader/src/create_zip.sh b/BlockBlobReader/src/create_zip.sh index 7b35d183..f85c9f7e 100755 --- a/BlockBlobReader/src/create_zip.sh +++ b/BlockBlobReader/src/create_zip.sh @@ -80,7 +80,7 @@ else fi echo "creating zip" -version="4.1.2" +version="4.1.4" producer_zip_file="taskproducer$version.zip" consumer_zip_file="taskconsumer$version.zip" dlqprocessor_zip_file="dlqprocessor$version.zip" diff --git a/BlockBlobReader/target/consumer_build/BlobTaskConsumer/index.js b/BlockBlobReader/target/consumer_build/BlobTaskConsumer/index.js index 9aeaf4fd..d5480d52 100644 --- a/BlockBlobReader/target/consumer_build/BlobTaskConsumer/index.js +++ b/BlockBlobReader/target/consumer_build/BlobTaskConsumer/index.js @@ -195,8 +195,9 @@ function getRowKey(metadata) { async function setAppendBlobOffset(context, serviceBusTask, newOffset) { + let rowKey = ""; try { - let rowKey = getRowKey(serviceBusTask); + rowKey = getRowKey(serviceBusTask); // Todo: this should be atomic update if other request decreases offset it shouldn't allow context.log.verbose("Attempting to update offset row: %s from: %d to: %d", rowKey, serviceBusTask.startByte, newOffset); let entity = { @@ -208,7 +209,7 @@ async function setAppendBlobOffset(context, serviceBusTask, newOffset) { containerName: serviceBusTask.containerName, storageName: serviceBusTask.storageName } - var updateResult = await azureTableClient.updateEntity(entity, "Merge"); + let updateResult = await azureTableClient.updateEntity(entity, "Merge"); context.log.verbose("Updated offset result: %s row: %s from: %d to: %d", JSON.stringify(updateResult), rowKey, serviceBusTask.startByte, newOffset); } catch (error) { context.log.error(`Error - Failed to update OffsetMap table, error: ${JSON.stringify(error)}, rowKey: ${rowKey}, newOffset: ${newOffset}`) diff --git a/BlockBlobReader/target/dlqprocessor_build/DLQTaskConsumer/index.js b/BlockBlobReader/target/dlqprocessor_build/DLQTaskConsumer/index.js index 9aeaf4fd..d5480d52 100644 --- a/BlockBlobReader/target/dlqprocessor_build/DLQTaskConsumer/index.js +++ b/BlockBlobReader/target/dlqprocessor_build/DLQTaskConsumer/index.js @@ -195,8 +195,9 @@ function getRowKey(metadata) { async function setAppendBlobOffset(context, serviceBusTask, newOffset) { + let rowKey = ""; try { - let rowKey = getRowKey(serviceBusTask); + rowKey = getRowKey(serviceBusTask); // Todo: this should be atomic update if other request decreases offset it shouldn't allow context.log.verbose("Attempting to update offset row: %s from: %d to: %d", rowKey, serviceBusTask.startByte, newOffset); let entity = { @@ -208,7 +209,7 @@ async function setAppendBlobOffset(context, serviceBusTask, newOffset) { containerName: serviceBusTask.containerName, storageName: serviceBusTask.storageName } - var updateResult = await azureTableClient.updateEntity(entity, "Merge"); + let updateResult = await azureTableClient.updateEntity(entity, "Merge"); context.log.verbose("Updated offset result: %s row: %s from: %d to: %d", JSON.stringify(updateResult), rowKey, serviceBusTask.startByte, newOffset); } catch (error) { context.log.error(`Error - Failed to update OffsetMap table, error: ${JSON.stringify(error)}, rowKey: ${rowKey}, newOffset: ${newOffset}`) diff --git a/BlockBlobReader/tests/blob_fixtures.json b/BlockBlobReader/tests/blob_fixtures.json index 7e040b2d..0d56daca 100644 --- a/BlockBlobReader/tests/blob_fixtures.json +++ b/BlockBlobReader/tests/blob_fixtures.json @@ -1 +1 @@ -{"records":[{"time":"2024-06-25T05:59:39.5591376Z","systemId":"26c09512-4912-48d1-9815-f70c8cf098f1","macAddress":"000D3A1D6249","category":"NetworkSecurityGroupFlowEvent","resourceId":"/SUBSCRIPTIONS/C088DC46-D692-42AD-A4B6-9A542D28AD2A/RESOURCEGROUPS/MARLOWIN11OTELTEST_GROUP/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/MARLOWIN11OTELTEST-NSG","operationName":"NetworkSecurityGroupFlowEvents","properties":{"Version":2,"flows":[{"rule":"DefaultRule_AllowInternetOutBound","flows":[{"mac":"000D3A1D6249","flowTuples":["1719295118,10.0.0.4,20.42.65.89,49867,443,T,O,A,B,,,,","1719295119,10.0.0.4,13.95.31.18,49868,443,T,O,A,B,,,,","1719295122,10.0.0.4,20.97.190.215,49865,443,T,O,A,E,43,32871,0,0","1719295123,10.0.0.4,23.222.201.247,49798,443,T,O,A,E,8,1371,0,0","1719295123,10.0.0.4,13.89.179.11,49869,443,T,O,A,B,,,,","1719295124,10.0.0.4,23.222.201.247,49803,443,T,O,A,E,1,66,0,0","1719295124,10.0.0.4,23.222.201.247,49765,443,T,O,A,E,18,4574,0,0","1719295124,10.0.0.4,23.222.201.247,49761,443,T,O,A,E,13,2809,0,0","1719295124,10.0.0.4,20.42.65.89,49867,443,T,O,A,E,16,6566,0,0","1719295124,10.0.0.4,23.222.201.247,49807,443,T,O,A,E,10,1460,0,0","1719295124,10.0.0.4,23.222.201.247,49799,443,T,O,A,E,10,1490,0,0","1719295125,10.0.0.4,23.222.201.247,49801,443,T,O,A,E,10,1490,0,0","1719295125,10.0.0.4,23.222.201.247,49802,443,T,O,A,E,10,1497,0,0","1719295125,10.0.0.4,52.247.240.158,49768,443,T,O,A,E,14,2126,0,0","1719295127,10.0.0.4,13.95.31.18,49868,443,T,O,A,E,1,66,0,0","1719295128,10.0.0.4,23.221.242.90,49773,443,T,O,A,E,9,1159,0,0","1719295129,10.0.0.4,23.221.242.90,49771,443,T,O,A,E,11,1226,0,0","1719295130,10.0.0.4,23.221.242.90,49772,443,T,O,A,E,11,1201,0,0","1719295130,10.0.0.4,13.89.179.11,49869,443,T,O,A,E,13,3074,0,0","1719295133,10.0.0.4,23.221.242.90,49774,443,T,O,A,E,51,5880,0,0","1719295134,10.0.0.4,40.79.154.80,49871,443,T,O,A,B,,,,","1719295136,10.0.0.4,20.150.90.100,49787,443,T,O,A,E,8,1555,0,0","1719295145,10.0.0.4,20.7.47.135,49873,443,T,O,A,B,,,,","1719295145,10.0.0.4,23.222.201.169,49874,443,T,O,A,B,,,,","1719295145,10.0.0.4,23.222.201.247,49875,443,T,O,A,B,,,,","1719295145,10.0.0.4,13.107.4.50,49876,80,T,O,A,B,,,,","1719295146,10.0.0.4,23.215.0.45,49877,80,T,O,A,B,,,,","1719295146,10.0.0.4,23.215.0.47,49878,80,T,O,A,B,,,,","1719295147,10.0.0.4,13.107.42.16,49880,443,T,O,A,B,,,,","1719295149,10.0.0.4,13.107.42.16,49882,443,T,O,A,B,,,,","1719295149,10.0.0.4,13.107.42.16,49883,443,T,O,A,B,,,,","1719295149,10.0.0.4,13.107.4.50,49884,80,T,O,A,B,,,,","1719295149,10.0.0.4,20.7.47.135,49885,443,T,O,A,B,,,,","1719295150,10.0.0.4,146.75.38.172,49886,80,T,O,A,B,,,,","1719295150,10.0.0.4,146.75.38.172,49887,80,T,O,A,B,,,,","1719295151,10.0.0.4,23.222.201.247,49755,443,T,O,A,E,16,4774,0,0","1719295152,10.0.0.4,44.217.84.165,49888,443,T,O,A,B,,,,","1719295152,10.0.0.4,13.107.42.16,49889,443,T,O,A,B,,,,","1719295152,10.0.0.4,23.222.201.169,49829,443,T,O,A,E,10,1256,0,0","1719295153,10.0.0.4,23.222.201.247,49797,443,T,O,A,E,12,2222,0,0","1719295155,10.0.0.4,23.222.201.247,49800,443,T,O,A,E,12,2208,0,0","1719295156,10.0.0.4,54.210.38.176,49855,443,T,O,A,E,31,21663,0,0","1719295158,10.0.0.4,20.7.47.135,49885,443,T,O,A,E,15,3046,0,0","1719295158,10.0.0.4,20.7.47.135,49890,443,T,O,A,B,,,,","1719295158,10.0.0.4,23.222.201.247,49891,443,T,O,A,B,,,,","1719295158,10.0.0.4,146.75.38.172,49892,80,T,O,A,B,,,,","1719295158,10.0.0.4,146.75.38.172,49893,80,T,O,A,B,,,,","1719295158,10.0.0.4,52.143.124.236,49894,443,T,O,A,B,,,,","1719295160,10.0.0.4,13.107.42.16,49889,443,T,O,A,E,12,2865,0,0","1719295160,10.0.0.4,20.150.79.68,49853,443,T,O,A,C,9,1615,29,39306","1719295160,10.0.0.4,20.209.147.193,49859,443,T,O,A,C,2855,207549,11272,15636810","1719295160,10.0.0.4,23.222.201.247,49806,443,T,O,A,C,14,2801,15,8454","1719295160,10.0.0.4,13.107.42.16,49883,443,T,O,A,C,11,2650,18,8858","1719295160,10.0.0.4,146.75.38.172,49893,80,T,O,A,C,1207,93006,42132,62600711","1719295160,10.0.0.4,23.221.241.211,49702,80,T,O,A,C,8,4180,6,1018","1719295160,10.0.0.4,13.71.55.58,49699,443,T,O,A,C,12,1653,14,10592","1719295160,10.0.0.4,168.61.215.74,123,123,U,O,A,C,1,90,1,90","1719295160,10.0.0.4,13.107.4.50,49690,80,T,O,A,C,1,66,4,742","1719295160,10.0.0.4,40.79.154.80,49863,443,T,O,A,C,15,7119,15,7986","1719295160,10.0.0.4,23.222.201.247,49753,443,T,O,A,C,10,1256,10,8103","1719295160,10.0.0.4,13.107.4.50,49876,80,T,O,A,C,7,1338,7,3251","1719295160,10.0.0.4,40.71.12.253,49849,443,T,O,A,C,7,3444,7,6124","1719295160,10.0.0.4,20.7.47.135,49890,443,T,O,A,C,1,66,1,66","1719295160,10.0.0.4,23.215.0.45,49877,80,T,O,A,C,1,74,1,74","1719295160,10.0.0.4,20.10.31.115,49823,443,T,O,A,C,12,2699,13,5576","1719295160,10.0.0.4,13.107.4.50,49838,80,T,O,A,C,1902,125768,24314,35935375","1719295160,10.0.0.4,13.107.42.16,49880,443,T,O,A,C,14,3182,20,8866","1719295160,10.0.0.4,23.222.201.247,49891,443,T,O,A,C,1,66,1,66","1719295160,10.0.0.4,13.107.4.50,49760,80,T,O,A,C,65,4233,643,954258","1719295160,10.0.0.4,52.167.164.252,49752,443,T,O,A,C,12,1367,9,3679","1719295160,10.0.0.4,40.71.12.253,49846,443,T,O,A,C,6,3164,10,7156","1719295160,10.0.0.4,40.71.12.253,49851,443,T,O,A,C,7,3476,8,7020","1719295160,10.0.0.4,52.148.179.131,49839,443,T,O,A,C,12,2032,9,3606","1719295160,10.0.0.4,44.217.84.165,49888,443,T,O,A,C,1,66,18,7272","1719295160,10.0.0.4,20.150.79.68,49847,443,T,O,A,C,30,4953,189,276793","1719295160,10.0.0.4,20.150.79.68,49852,443,T,O,A,C,7,1510,16,19883","1719295160,10.0.0.4,52.191.219.104,49725,443,T,O,A,C,11,1365,11,4933","1719295160,10.0.0.4,146.75.38.172,49887,80,T,O,A,C,589,42831,6119,9019179","1719295160,10.0.0.4,13.107.42.16,49882,443,T,O,A,C,14,2829,20,8978","1719295160,10.0.0.4,13.107.4.50,49762,80,T,O,A,C,442,28810,5009,7407213","1719295160,10.0.0.4,146.75.38.172,49892,80,T,O,A,C,2482,190086,42880,63699530","1719295160,10.0.0.4,20.12.23.50,49706,443,T,O,A,C,13,1524,34,40005","1719295160,10.0.0.4,52.200.189.154,49700,443,T,O,A,C,198,110395,115,38389","1719295160,10.0.0.4,20.42.73.174,49784,443,T,O,A,C,10,5535,10,8419","1719295160,10.0.0.4,40.71.12.253,49848,443,T,O,A,C,6,3164,7,6124","1719295160,10.0.0.4,23.222.201.247,49875,443,T,O,A,C,12,2746,11,9190","1719295160,10.0.0.4,52.143.124.236,49894,443,T,O,A,C,1,66,1,66","1719295160,10.0.0.4,20.10.31.115,49734,443,T,O,A,C,17,3674,18,7112","1719295160,10.0.0.4,20.150.79.68,49854,443,T,O,A,C,9,1653,30,39831","1719295160,10.0.0.4,13.107.4.50,49837,80,T,O,A,C,1977,130617,24568,36315436","1719295160,10.0.0.4,52.140.118.28,49796,443,T,O,A,C,12,1990,10,4844","1719295160,10.0.0.4,20.209.85.129,49857,443,T,O,A,C,14,1391,6,5996","1719295160,10.0.0.4,23.222.201.169,49874,443,T,O,A,C,1,66,1,66","1719295160,10.0.0.4,40.71.12.253,49845,443,T,O,A,C,10,5058,12,8107","1719295160,10.0.0.4,52.137.125.63,49836,443,T,O,A,C,12,2021,10,3665","1719295160,10.0.0.4,40.71.12.253,49850,443,T,O,A,C,6,3164,8,7005","1719295160,10.0.0.4,52.191.219.104,49724,443,T,O,A,C,12,2559,10,4518","1719295160,10.0.0.4,52.183.220.149,49739,443,T,O,A,C,15,2837,25,26236","1719295160,10.0.0.4,23.222.201.247,49804,443,T,O,A,C,13,3249,18,8659","1719295160,10.0.0.4,13.107.4.50,49756,80,T,O,A,C,14,2990,23,19303","1719295160,10.0.0.4,146.75.38.172,49886,80,T,O,A,C,649,46779,6714,9888359","1719295160,10.0.0.4,13.107.4.50,49766,80,T,O,A,C,870,56786,9652,14273018","1719295160,10.0.0.4,23.215.0.47,49878,80,T,O,A,C,130,9025,420,614202","1719295160,10.0.0.4,20.7.47.135,49873,443,T,O,A,C,1,66,1,66","1719295160,10.0.0.4,52.168.117.168,49861,443,T,O,A,C,12,2999,12,5879","1719295160,10.0.0.4,40.79.154.80,49871,443,T,O,A,C,7,4075,7,6194","1719295160,10.0.0.4,52.205.149.247,49692,443,T,O,A,C,38,4012,27,14093","1719295160,10.0.0.4,52.191.219.104,49779,443,T,O,A,C,12,2831,11,5304","1719295160,10.0.0.4,13.107.4.50,49884,80,T,O,A,C,10,882,53,75350","1719295165,10.0.0.4,23.222.201.247,49806,443,T,O,A,E,0,0,0,0","1719295165,10.0.0.4,23.222.201.247,49804,443,T,O,A,E,0,0,0,0"]}]},{"rule":"DefaultRule_DenyAllInBound","flows":[{"mac":"000D3A1D6249","flowTuples":["1719295119,109.120.186.165,10.0.0.4,57451,2455,T,I,D,B,,,,","1719295122,162.216.149.9,10.0.0.4,49530,48668,T,I,D,B,,,,","1719295131,167.94.145.17,10.0.0.4,3055,104,T,I,D,B,,,,","1719295142,162.216.150.89,10.0.0.4,49309,45663,T,I,D,B,,,,","1719295154,147.185.133.218,10.0.0.4,51463,9329,T,I,D,B,,,,","1719295158,162.216.150.110,10.0.0.4,54867,19854,T,I,D,B,,,,","1719295163,152.32.207.172,10.0.0.4,43160,2435,T,I,D,B,,,,","1719295169,170.39.216.2,10.0.0.4,49603,8728,T,I,D,B,,,,","1719295173,101.53.240.140,10.0.0.4,15430,23,T,I,D,B,,,,"]}]}]}}]} \ No newline at end of file +{"records":[{"time":"2024-08-19T18:00:06.8944384Z","systemId":"05cdbee8-43c6-458e-b53c-d707f958c66b","macAddress":"6045BDEC2532","category":"NetworkSecurityGroupFlowEvent","resourceId":"/SUBSCRIPTIONS/11111111-2222-3333-A4B6-000000000000/RESOURCEGROUPS/VM-TEST/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/TEST-NSG","operationName":"NetworkSecurityGroupFlowEvents","properties":{"Version":2,"flows":[{"rule":"DefaultRule_AllowInternetOutBound","flows":[{"mac":"6045BDEC2532","flowTuples":["1724090346,10.1.0.4,20.98.192.237,35620,443,T,O,A,E,1,74,0,0","1724090347,10.1.0.4,20.98.192.237,35636,443,T,O,A,E,1,74,0,0","1724090347,10.1.0.4,20.98.192.237,35640,443,T,O,A,E,1,74,0,0","1724090348,10.1.0.4,52.182.143.213,41968,443,T,O,A,E,1,74,0,0","1724090350,10.1.0.4,20.42.73.216,51236,443,T,O,A,B,,,,","1724090350,10.1.0.4,20.42.73.216,51246,443,T,O,A,B,,,,","1724090355,10.1.0.4,20.42.73.216,51236,443,T,O,A,E,9,4817,0,0","1724090355,10.1.0.4,20.42.73.216,51246,443,T,O,A,E,8,4076,0,0","1724090361,10.1.0.4,20.10.127.193,46456,443,T,O,A,B,,,,","1724090367,10.1.0.4,20.10.127.193,46456,443,T,O,A,E,1,74,0,0","1724090383,10.1.0.4,40.79.154.80,33338,443,T,O,A,E,0,0,0,0","1724090391,10.1.0.4,20.10.127.193,53884,443,T,O,A,B,,,,","1724090397,10.1.0.4,20.10.127.193,53884,443,T,O,A,E,1,74,0,0","1724090403,10.1.0.4,20.42.73.25,59180,443,T,O,A,B,,,,"]}]},{"rule":"DefaultRule_DenyAllInBound","flows":[{"mac":"6045BDEC2532","flowTuples":["1724090369,35.203.211.209,10.1.0.4,55740,38405,T,I,D,B,,,,","1724090383,162.216.150.90,10.1.0.4,54750,22083,T,I,D,B,,,,","1724090389,203.55.81.13,10.1.0.4,34466,8728,T,I,D,B,,,,","1724090394,35.203.211.172,10.1.0.4,57198,2424,T,I,D,B,,,,"]}]}]}},{"time":"2024-08-19T18:01:06.8931133Z","systemId":"05cdbee8-43c6-458e-b53c-d707f958c66b","macAddress":"6045BDEC2532","category":"NetworkSecurityGroupFlowEvent","resourceId":"/SUBSCRIPTIONS/11111111-2222-3333-A4B6-000000000000/RESOURCEGROUPS/VM-TEST/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/TEST-NSG","operationName":"NetworkSecurityGroupFlowEvents","properties":{"Version":2,"flows":[{"rule":"DefaultRule_AllowInternetOutBound","flows":[{"mac":"6045BDEC2532","flowTuples":["1724090409,10.1.0.4,20.42.73.25,59180,443,T,O,A,E,14,5552,0,0","1724090410,10.1.0.4,20.42.73.204,40912,443,T,O,A,B,,,,","1724090410,10.1.0.4,20.42.73.204,40926,443,T,O,A,B,,,,","1724090415,10.1.0.4,20.42.73.204,40926,443,T,O,A,E,8,4077,0,0","1724090415,10.1.0.4,20.42.73.204,40912,443,T,O,A,E,4,1312,0,0","1724090421,10.1.0.4,20.10.127.193,33612,443,T,O,A,B,,,,","1724090427,10.1.0.4,20.10.127.193,33612,443,T,O,A,E,1,74,0,0","1724090438,10.1.0.4,40.79.154.80,37650,443,T,O,A,E,0,0,0,0","1724090451,10.1.0.4,20.10.127.193,37246,443,T,O,A,B,,,,","1724090457,10.1.0.4,20.10.127.193,37246,443,T,O,A,E,1,74,0,0"]}]},{"rule":"DefaultRule_DenyAllInBound","flows":[{"mac":"6045BDEC2532","flowTuples":["1724090408,79.124.62.134,10.1.0.4,57919,13387,T,I,D,B,,,,","1724090415,35.203.210.127,10.1.0.4,49554,9497,T,I,D,B,,,,","1724090418,193.163.125.185,10.1.0.4,47827,6688,T,I,D,B,,,,","1724090420,35.203.211.153,10.1.0.4,53924,18094,T,I,D,B,,,,","1724090423,194.26.29.198,10.1.0.4,50964,1849,T,I,D,B,,,,","1724090423,193.41.206.156,10.1.0.4,51447,8728,T,I,D,B,,,,","1724090424,172.232.216.172,10.1.0.4,48072,2087,T,I,D,B,,,,","1724090426,147.78.47.19,10.1.0.4,59518,33897,T,I,D,B,,,,","1724090426,79.110.62.76,10.1.0.4,45259,57162,T,I,D,B,,,,","1724090443,162.216.149.73,10.1.0.4,54860,3358,T,I,D,B,,,,","1724090444,35.203.210.93,10.1.0.4,56514,12128,T,I,D,B,,,,","1724090445,162.216.150.109,10.1.0.4,55489,491,T,I,D,B,,,,","1724090447,79.110.62.76,10.1.0.4,45259,61014,T,I,D,B,,,,","1724090447,194.26.29.111,10.1.0.4,44298,6453,T,I,D,B,,,,","1724090449,79.124.62.126,10.1.0.4,57431,46209,T,I,D,B,,,,","1724090451,167.94.145.84,10.1.0.4,41113,13802,T,I,D,B,,,,","1724090456,45.67.221.140,10.1.0.4,49581,2375,T,I,D,B,,,,","1724090460,78.128.114.2,10.1.0.4,47535,970,T,I,D,B,,,,"]}]}]}},{"time":"2024-08-19T18:02:06.9579599Z","systemId":"05cdbee8-43c6-458e-b53c-d707f958c66b","macAddress":"6045BDEC2532","category":"NetworkSecurityGroupFlowEvent","resourceId":"/SUBSCRIPTIONS/11111111-2222-3333-A4B6-000000000000/RESOURCEGROUPS/VM-TEST/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/TEST-NSG","operationName":"NetworkSecurityGroupFlowEvents","properties":{"Version":2,"flows":[{"rule":"DefaultRule_AllowInternetOutBound","flows":[{"mac":"6045BDEC2532","flowTuples":["1724090470,10.1.0.4,20.42.73.204,60386,443,T,O,A,B,,,,","1724090475,10.1.0.4,20.42.73.204,60386,443,T,O,A,E,9,4143,0,0","1724090481,10.1.0.4,20.10.127.193,54556,443,T,O,A,B,,,,","1724090487,10.1.0.4,20.10.127.193,54556,443,T,O,A,E,1,74,0,0","1724090511,10.1.0.4,20.10.127.193,33142,443,T,O,A,B,,,,","1724090517,10.1.0.4,20.10.127.193,33142,443,T,O,A,E,1,74,0,0","1724090519,10.1.0.4,20.10.127.192,48368,443,T,O,A,C,12,5434,11,6342"]}]},{"rule":"DefaultRule_DenyAllInBound","flows":[{"mac":"6045BDEC2532","flowTuples":["1724090465,113.26.224.60,10.1.0.4,38936,23,T,I,D,B,,,,","1724090469,4.151.220.65,10.1.0.4,43716,55554,T,I,D,B,,,,","1724090471,79.124.62.74,10.1.0.4,57235,22500,T,I,D,B,,,,","1724090493,79.124.62.126,10.1.0.4,57431,26295,T,I,D,B,,,,","1724090494,89.248.165.188,10.1.0.4,56318,63222,T,I,D,B,,,,","1724090501,83.222.190.110,10.1.0.4,40669,27627,T,I,D,B,,,,","1724090501,120.26.45.136,10.1.0.4,33148,2222,T,I,D,B,,,,","1724090519,35.203.210.90,10.1.0.4,52043,10092,T,I,D,B,,,,","1724090520,89.248.165.203,10.1.0.4,55704,62396,T,I,D,B,,,,","1724090523,194.26.29.221,10.1.0.4,44465,8957,T,I,D,B,,,,"]}]},{"rule":"UserRule_SSH","flows":[{"mac":"6045BDEC2532","flowTuples":["1724090463,120.26.45.136,10.1.0.4,58136,22,T,I,A,B,,,,"]}]}]}},{"time":"2024-08-19T18:03:06.9556111Z","systemId":"05cdbee8-43c6-458e-b53c-d707f958c66b","macAddress":"6045BDEC2532","category":"NetworkSecurityGroupFlowEvent","resourceId":"/SUBSCRIPTIONS/11111111-2222-3333-A4B6-000000000000/RESOURCEGROUPS/VM-TEST/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/TEST-NSG","operationName":"NetworkSecurityGroupFlowEvents","properties":{"Version":2,"flows":[{"rule":"DefaultRule_AllowInternetOutBound","flows":[{"mac":"6045BDEC2532","flowTuples":["1724090530,10.1.0.4,20.42.73.216,38250,443,T,O,A,B,,,,","1724090542,10.1.0.4,20.10.127.193,49314,443,T,O,A,B,,,,","1724090547,10.1.0.4,20.10.127.193,49314,443,T,O,A,E,1,74,0,0","1724090564,10.1.0.4,20.189.173.16,55584,443,T,O,A,B,,,,","1724090570,10.1.0.4,20.42.73.216,60778,443,T,O,A,B,,,,","1724090570,10.1.0.4,20.189.173.16,55584,443,T,O,A,E,1,74,0,0","1724090572,10.1.0.4,20.10.127.193,32830,443,T,O,A,B,,,,","1724090575,10.1.0.4,20.42.73.216,60778,443,T,O,A,E,8,4597,0,0","1724090577,10.1.0.4,20.10.127.193,32830,443,T,O,A,E,1,74,0,0"]}]},{"rule":"DefaultRule_DenyAllInBound","flows":[{"mac":"6045BDEC2532","flowTuples":["1724090534,83.222.190.110,10.1.0.4,40669,31009,T,I,D,B,,,,","1724090546,89.248.163.73,10.1.0.4,56663,63871,T,I,D,B,,,,","1724090548,147.185.133.72,10.1.0.4,56240,9678,T,I,D,B,,,,","1724090568,60.205.137.128,10.1.0.4,49169,3389,T,I,D,B,,,,","1724090570,35.203.210.80,10.1.0.4,50140,45277,T,I,D,B,,,,","1724090570,80.94.95.249,10.1.0.4,61000,14787,T,I,D,B,,,,","1724090580,79.110.62.76,10.1.0.4,45259,61890,T,I,D,B,,,,","1724090583,79.124.62.134,10.1.0.4,57919,58022,T,I,D,B,,,,"]}]}]}},{"time":"2024-08-19T18:04:06.9574224Z","systemId":"05cdbee8-43c6-458e-b53c-d707f958c66b","macAddress":"6045BDEC2532","category":"NetworkSecurityGroupFlowEvent","resourceId":"/SUBSCRIPTIONS/11111111-2222-3333-A4B6-000000000000/RESOURCEGROUPS/VM-TEST/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/TEST-NSG","operationName":"NetworkSecurityGroupFlowEvents","properties":{"Version":2,"flows":[{"rule":"DefaultRule_AllowInternetOutBound","flows":[{"mac":"6045BDEC2532","flowTuples":["1724090590,10.1.0.4,20.42.73.216,52834,443,T,O,A,B,,,,","1724090595,10.1.0.4,20.42.73.216,52834,443,T,O,A,E,13,4431,0,0","1724090602,10.1.0.4,20.10.127.193,51828,443,T,O,A,B,,,,","1724090632,10.1.0.4,20.10.127.193,33214,443,T,O,A,B,,,,","1724090637,10.1.0.4,20.10.127.193,33214,443,T,O,A,E,1,74,0,0"]}]},{"rule":"DefaultRule_DenyAllInBound","flows":[{"mac":"6045BDEC2532","flowTuples":["1724090589,193.163.125.211,10.1.0.4,54994,322,T,I,D,B,,,,","1724090613,162.216.150.14,10.1.0.4,51561,46747,T,I,D,B,,,,","1724090614,35.203.211.187,10.1.0.4,50814,2121,T,I,D,B,,,,","1724090616,89.248.163.197,10.1.0.4,46403,36101,T,I,D,B,,,,","1724090619,205.210.31.135,10.1.0.4,50158,2323,T,I,D,B,,,,","1724090620,35.203.211.79,10.1.0.4,53688,7903,T,I,D,B,,,,","1724090628,195.230.103.248,10.1.0.4,47033,6000,T,I,D,B,,,,","1724090634,172.232.195.38,10.1.0.4,53272,2087,T,I,D,B,,,,"]}]}]}},{"time":"2024-08-19T18:05:07.0426683Z","systemId":"05cdbee8-43c6-458e-b53c-d707f958c66b","macAddress":"6045BDEC2532","category":"NetworkSecurityGroupFlowEvent","resourceId":"/SUBSCRIPTIONS/11111111-2222-3333-A4B6-000000000000/RESOURCEGROUPS/VM-TEST/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/TEST-NSG","operationName":"NetworkSecurityGroupFlowEvents","properties":{"Version":2,"flows":[{"rule":"DefaultRule_AllowInternetOutBound","flows":[{"mac":"6045BDEC2532","flowTuples":["1724090647,10.1.0.4,40.79.154.85,52566,443,T,O,A,B,,,,","1724090649,10.1.0.4,20.189.173.11,49436,443,T,O,A,B,,,,","1724090650,10.1.0.4,20.42.73.216,34232,443,T,O,A,B,,,,","1724090654,10.1.0.4,40.79.154.85,52566,443,T,O,A,E,9,5156,0,0","1724090655,10.1.0.4,20.42.73.216,34232,443,T,O,A,E,8,4077,0,0","1724090656,10.1.0.4,20.189.173.11,49436,443,T,O,A,E,11,3498,0,0","1724090662,10.1.0.4,20.10.127.193,33420,443,T,O,A,B,,,,","1724090667,10.1.0.4,20.10.127.193,33420,443,T,O,A,E,1,74,0,0","1724090692,10.1.0.4,20.10.127.193,45366,443,T,O,A,B,,,,","1724090697,10.1.0.4,20.10.127.193,45366,443,T,O,A,E,1,74,0,0","1724090703,10.1.0.4,20.189.173.18,40424,443,T,O,A,B,,,,"]}]},{"rule":"DefaultRule_DenyAllInBound","flows":[{"mac":"6045BDEC2532","flowTuples":["1724090644,35.203.211.55,10.1.0.4,50408,3392,T,I,D,B,,,,","1724090650,194.26.29.198,10.1.0.4,50964,1189,T,I,D,B,,,,","1724090655,35.203.211.73,10.1.0.4,53700,2226,T,I,D,B,,,,","1724090655,35.203.211.33,10.1.0.4,54262,2021,T,I,D,B,,,,","1724090674,35.203.211.14,10.1.0.4,56635,9970,T,I,D,B,,,,","1724090680,79.110.62.76,10.1.0.4,45259,54091,T,I,D,B,,,,","1724090685,147.185.133.87,10.1.0.4,53320,222,T,I,D,B,,,,","1724090702,198.211.106.106,10.1.0.4,40531,27017,T,I,D,B,,,,","1724090703,79.110.62.76,10.1.0.4,45259,61699,T,I,D,B,,,,"]}]}]}},{"time":"2024-08-19T18:06:07.0553830Z","systemId":"05cdbee8-43c6-458e-b53c-d707f958c66b","macAddress":"6045BDEC2532","category":"NetworkSecurityGroupFlowEvent","resourceId":"/SUBSCRIPTIONS/11111111-2222-3333-A4B6-000000000000/RESOURCEGROUPS/VM-TEST/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/TEST-NSG","operationName":"NetworkSecurityGroupFlowEvents","properties":{"Version":2,"flows":[{"rule":"DefaultRule_AllowInternetOutBound","flows":[{"mac":"6045BDEC2532","flowTuples":["1724090709,10.1.0.4,20.189.173.18,40424,443,T,O,A,E,1,74,0,0","1724090710,10.1.0.4,20.42.73.216,34592,443,T,O,A,B,,,,","1724090710,10.1.0.4,20.42.73.216,34596,443,T,O,A,B,,,,","1724090715,10.1.0.4,20.42.73.216,34592,443,T,O,A,E,8,4775,0,0","1724090715,10.1.0.4,20.42.73.216,34596,443,T,O,A,E,8,3560,0,0","1724090722,10.1.0.4,20.10.127.193,43580,443,T,O,A,B,,,,","1724090727,10.1.0.4,20.10.127.193,43580,443,T,O,A,E,1,74,0,0","1724090752,10.1.0.4,20.10.127.193,37524,443,T,O,A,B,,,,","1724090757,10.1.0.4,20.10.127.193,37524,443,T,O,A,E,1,74,0,0"]}]},{"rule":"DefaultRule_DenyAllInBound","flows":[{"mac":"6045BDEC2532","flowTuples":["1724090713,194.26.29.113,10.1.0.4,44599,9791,T,I,D,B,,,,","1724090716,141.98.11.128,10.1.0.4,54527,8728,T,I,D,B,,,,","1724090720,79.110.62.76,10.1.0.4,45259,63991,T,I,D,B,,,,","1724090722,206.168.35.169,10.1.0.4,51854,1311,T,I,D,B,,,,","1724090730,35.203.211.195,10.1.0.4,55697,17004,T,I,D,B,,,,","1724090730,147.185.133.253,10.1.0.4,54890,9823,T,I,D,B,,,,","1724090743,35.203.211.197,10.1.0.4,52346,4000,T,I,D,B,,,,","1724090748,162.216.149.232,10.1.0.4,53271,9905,T,I,D,B,,,,","1724090758,162.216.150.180,10.1.0.4,56902,9987,T,I,D,B,,,,","1724090759,198.235.24.133,10.1.0.4,51697,10250,T,I,D,B,,,,","1724090761,79.110.62.76,10.1.0.4,45259,61253,T,I,D,B,,,,"]}]}]}},{"time":"2024-08-19T18:07:07.0570871Z","systemId":"05cdbee8-43c6-458e-b53c-d707f958c66b","macAddress":"6045BDEC2532","category":"NetworkSecurityGroupFlowEvent","resourceId":"/SUBSCRIPTIONS/11111111-2222-3333-A4B6-000000000000/RESOURCEGROUPS/VM-TEST/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/TEST-NSG","operationName":"NetworkSecurityGroupFlowEvents","properties":{"Version":2,"flows":[{"rule":"DefaultRule_AllowInternetOutBound","flows":[{"mac":"6045BDEC2532","flowTuples":["1724090766,10.1.0.4,40.79.154.85,57104,443,T,O,A,B,,,,","1724090770,10.1.0.4,40.71.12.254,42024,443,T,O,A,B,,,,","1724090771,10.1.0.4,20.10.127.192,48368,443,T,O,A,E,0,0,0,0","1724090771,10.1.0.4,40.79.154.85,57104,443,T,O,A,E,9,4866,0,0","1724090775,10.1.0.4,40.71.12.254,42024,443,T,O,A,E,8,4078,0,0","1724090782,10.1.0.4,20.10.127.192,42714,443,T,O,A,B,,,,","1724090787,10.1.0.4,20.10.127.192,42714,443,T,O,A,E,1,74,0,0","1724090803,10.1.0.4,20.42.73.216,38250,443,T,O,A,E,5,1378,0,0","1724090812,10.1.0.4,20.10.127.193,32876,443,T,O,A,B,,,,","1724090819,10.1.0.4,20.10.127.193,32876,443,T,O,A,C,1,74,1,74","1724090819,10.1.0.4,20.10.127.193,51828,443,T,O,A,C,1,74,1,74"]}]},{"rule":"DefaultRule_DenyAllInBound","flows":[{"mac":"6045BDEC2532","flowTuples":["1724090767,89.248.165.195,10.1.0.4,55935,62758,T,I,D,B,,,,","1724090774,35.203.210.192,10.1.0.4,56948,9135,T,I,D,B,,,,","1724090780,35.203.210.71,10.1.0.4,51152,1977,T,I,D,B,,,,","1724090788,79.124.62.134,10.1.0.4,57919,21531,T,I,D,B,,,,","1724090789,79.110.62.76,10.1.0.4,45259,52977,T,I,D,B,,,,","1724090814,79.124.62.134,10.1.0.4,57919,52725,T,I,D,B,,,,","1724090819,35.203.211.151,10.1.0.4,50456,46900,T,I,D,B,,,,"]}]},{"rule":"UserRule_SSH","flows":[{"mac":"6045BDEC2532","flowTuples":["1724090812,209.38.19.52,10.1.0.4,56233,22,T,I,A,B,,,,"]}]}]}}]} \ No newline at end of file diff --git a/BlockBlobReader/tests/test_blobreader.py b/BlockBlobReader/tests/test_blobreader.py index c96f0500..0378fa8c 100644 --- a/BlockBlobReader/tests/test_blobreader.py +++ b/BlockBlobReader/tests/test_blobreader.py @@ -98,7 +98,7 @@ def get_full_testlog_file_name(self): if len(self.test_filename) > maxMetadataLength: expected_filename = self.test_filename[:60] + "..." + self.test_filename[-(60-len(file_ext)):] + file_ext else: - expected_filename = self.test_filename + expected_filename = self.test_filename + file_ext return expected_filename def test_03_func_logs(self): @@ -146,7 +146,7 @@ def test_03_func_logs(self): expected_record_count = { "blob": 15, "log": 10, - "json": 120, + "json": 153, "csv": 12 } record_count = record_excluded_by_filter_count = record_unsupported_extension_count = None @@ -184,7 +184,9 @@ def test_03_func_logs(self): def upload_message_in_service_bus(self): file_ext = f".{self.log_type}" test_filename = self.test_filename + file_ext - file_size = os.path.getsize(f"blob_fixtures{file_ext}") + with open(f"blob_fixtures{file_ext}", "r") as fp: + file_size = len(fp.read()) + triggerData = { "blobName": test_filename, "containerName": self.test_container_name, @@ -329,7 +331,8 @@ def insert_mock_json_in_BlobStorage(self): self.test_container_name, test_filename) for i, data_block in enumerate(self.get_json_data()): block_id = self.get_random_name() - file_bytes = json.dumps(data_block) + # removing spaces(added by json.loads) using separators + file_bytes = json.dumps(data_block, separators=(',', ':')) file_bytes = (file_bytes[1:-1] if i == 0 else "," + file_bytes[1:-1]).encode() self.block_blob_service.put_block(