reorganize file structure + fix sources bug in blank_columns.py

.gitignore

@@ -13,9 +13,9 @@
 sfm_pc/management/commands/data/
 country_data/
 wwic_download.zip
-data/wwic_download/*.csv
-data/wwic_download/*.geojson
-data/wwic_download/*.pdf
+data/wwic_download/countries/*.csv
+data/wwic_download/countries/*.geojson
+data/wwic_download/metadata/*.pdf
 
 /staticfiles
 bin/

data/processors/blank_columns.py

@@ -15,8 +15,20 @@
 stdout_csv.writeheader()
 
 for row in reader:
-    row.update({
-        f'{args.entity}:comments:admin': '',
-        f'{args.entity}:owner:admin': ''
-    })
+    comment_key = f'{args.entity}:comments:admin'
+    comments = row.get(comment_key)
+
+    if comments:
+        row.update({
+            comment_key: ''
+        })
+
+    owner_key = f'{args.entity}:owner:admin'
+    owner = row.get(owner_key)
+
+    if owner:
+        row.update({
+            owner_key: ''
+        })
+
     stdout_csv.writerow(row)

docket.mk

@@ -1,44 +1,45 @@
-.PHONY: sfm_pc/management/commands/country_data
+.PHONY: sfm_pc/management/commands/country_data data/wwic_download/countries
 
 DATA_ARCHIVE_BUCKET := $(shell cat configs/s3_config.json | jq -r '.data_archive_bucket')
 
 data_archive : wwic_download.zip
 	aws s3 cp $< s3://$(DATA_ARCHIVE_BUCKET)/
 
 .PHONY: wwic_download.zip
-wwic_download.zip : filtered_data data/wwic_download/sources.csv data/wwic_download/sfm_research_handbook.pdf
+wwic_download.zip : filtered_data data/wwic_download/metadata/sfm_research_handbook.pdf
 	cd data/wwic_download && zip -r ../../$@ .
 
 COUNTRY_NAMES=$(shell perl -pe "s/,/ /g" import_docket.csv | cut -d' ' -f5)
-ENTITIES=units.csv persons.csv incidents.csv locations.csv locations.geojson
+ENTITIES=units.csv persons.csv incidents.csv locations.csv locations.geojson sources.csv
 
 .PHONY : filtered_data
-filtered_data: $(foreach country,$(COUNTRY_NAMES),$(patsubst %,data/wwic_download/$(country)_%,$(ENTITIES)))
+filtered_data: $(foreach country,$(COUNTRY_NAMES),$(patsubst %,data/wwic_download/countries/$(country)_%,$(ENTITIES)))
 	echo "filtered csvs for entities"
 
 define filter_entity_data
 	$(shell csvgrep --columns $(1):status:admin --match 3 $< | \
 					python data/processors/blank_columns.py --entity $(1) > $@)
 endef
 
-data/wwic_download/%_units.csv : sfm_pc/management/commands/country_data/countries/%/units.csv
+data/wwic_download/countries/%_units.csv : sfm_pc/management/commands/country_data/countries/%/units.csv
 	$(call filter_entity_data,unit)
 
-data/wwic_download/%_persons.csv : sfm_pc/management/commands/country_data/countries/%/persons.csv
+data/wwic_download/countries/%_persons.csv : sfm_pc/management/commands/country_data/countries/%/persons.csv
 	$(call filter_entity_data,person)
 
-data/wwic_download/%_incidents.csv : sfm_pc/management/commands/country_data/countries/%/incidents.csv
+data/wwic_download/countries/%_incidents.csv : sfm_pc/management/commands/country_data/countries/%/incidents.csv
 	$(call filter_entity_data,incident)
 
-data/wwic_download/sources.csv : sfm_pc/management/commands/country_data/countries/sources.csv
+data/wwic_download/countries/%_sources.csv : sfm_pc/management/commands/country_data/countries/%/sources.csv
 	$(call filter_entity_data,source)
 
-data/wwic_download/%_locations.csv : sfm_pc/management/commands/country_data/countries/%/locations.csv
+data/wwic_download/countries/%_locations.csv : sfm_pc/management/commands/country_data/countries/%/locations.csv
 	cp $< $@
 
-data/wwic_download/%_locations.geojson : sfm_pc/management/commands/country_data/countries/%/locations.geojson
+data/wwic_download/countries/%_locations.geojson : sfm_pc/management/commands/country_data/countries/%/locations.geojson
 	cp $< $@
 
+.PHONY : data/wwic_download/metadata/sfm_research_handbook.pdf
 data/wwic_download/metadata/sfm_research_handbook.pdf : 
 	curl -o $@ https://help.securityforcemonitor.org/_/downloads/en/latest/pdf/
 

fixtures/import_docket.csv

@@ -1,26 +1,26 @@
 source_document_id,location_document_id,entitity_document_id,sfm:iso,sfm:country_name
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1ztOfGaQT3WDrq-BOjT0x5VErzgrWQ0Ku,19Hk6OD5AYjWQCUaTWPzkGm9sLqp4e5v_n667M0CgaQQ,ae,united-arab-emirates
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1HpIjYaH_iMCRQD1jP159VGz-2NL4nB5p,1n9NZXDwr9gH6qT1k8-zNz34aq2MncG1RAiGHRfUHfAc,bd,bangladesh
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1j8KgLnpjlnLy6bN4ozkwnBpkHUI6i3si,1IN8uZeR7WJbAmgPkY_QzzIir_cuba_Irzd_FtceUGyQ,bf,burkina-faso
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1mjTLMZ1la3zyiVQxLZ56sW497Sp8Lh5m,1tJXt33b5yNJajfTh2j6oXyWAduNaBRTy-XZON5MwspE,bh,bahrain
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1qZoQciglG1DOeEa3hh5iUvF7q4_bKOQl,1d0YaTdc1Esj0MVncM6cyhch6oT2VkWm7z_K4xEZELDs,eg,egypt
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1eZHw6k7xM7Z_ZNMnF0Wc5cjJuVyuOop3,1KCiLecZiMCzD7JXY4r_cYgcjr54aG3CnMOOsrrGfMoQ,jo,jordan
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1vnWgeTkq5TUyF7555F4renMJnl4WiFNy,1k9XBHbLM3mGDVpokgmV5H8T0anOSsuWVRX_5PeQ_1ik,kw,kuwait
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,13XnZOF7U0uwL3EP_QpdTVd1FUh3A3cwi,1CGvLuzRIBmuzKGknc3mUWISVLx9OwlpN22z6nFw3cLY,lr,liberia
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1fnTq_ONVxzVBbCyQy_-s6ngmadA6st70,1swzBv27-BOlMEcW3bW0MbS0nK30ra0eZjKXv8hQAEpQ,ma,morocco
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1syUOihIFKzk6QsLXd7XNUZIwzZZfAqEH,1sBW4eudBid8kOt-48XW6QQxNnR1NdXfoESudFyv0vMo,ml,mali
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1dU24WM8MAnqApFcBoYCiKPzPevebal6_,1Kt18on6vuUIWsuEr_LxJfpPw-WngE843sJjpeotupdo,mm,myanmar
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,17Zqej6mrCT3BBBKcKj7949qHyRCa-9SJ,1yv5Gt4TYFplbNHLg7ZJsb7ZVPfCE2xD1GINm2e52Wgs,mr,mauritania
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1nVWV5_1kGDwyWJ3PPqExKfchs3sAlEuh,1GKJIanAuch2j64-HDqInvfG46dWhULswPQNYbCC97y0,mx,mexico
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1qlHquI9EDz2lteBcjz_MheNLspg3mp_q,1KoVVqaPcuNz3Y7GLns8n3BnewwuNSxLK-qbY1dfMhG8,ne,niger
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1boFMPurqyxbfYBvfisRiROmzr8TuPI8j,1UgD7O9e5HSpj60tT4p73i0k2mMdoI8jtEZLwTUdF148,ng,nigeria
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1nMXXtFwJ3TqeynpKSW11uYAzihSMV8So,1cI_ahU1yEVQdHyhtrxUQ2LccaHNu9uOwwk_diclZgHc,np,nepal
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,19o8a1zwxUEYFxvZkqs2AwCyIm0oe_CPF,1BZWTt_ukmo9HxVvf9atN-9g_W7j57K3Q_mms0xAmN4U,ph,philippines
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1x4LjK_UWdxZm5EyNCupj7ikv7E-WMrkd,1uZtsbPTX9NVBrOCHLtdCum-2c2N9w4ALRGeo5miKCWE,qa,qatar
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1wSzKP9bsXB9w4U8frj4Y5kHrfV3C6Vi8,1PpAqUl5ijUAM_cHgxTIalMwsDnauEf2xojGSLQhGb1U,rw,rwanda
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1FLls5iHQD71Omy4VEzXYQ4HacMubzg8v,1zBVkLW_vR1kuJ5dEaXdh9hIYG4ExU7Be6pT9MpldUek,sa,saudi-arabia
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1tNS4yJYlh265zDy9rQnjAZSqSmBZxrRh,1pxX5spQL1oe7fcJHX-2jg-Q2bGe5T9HrOVUN528HEaI,sd,sudan
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1l3NE8P8Xi-1qGwqZcdVdvV3Hn1h4Bwjv,1YC3W5Q2EGwq0dPOuZn7rUu9nMKr5j9MrThBv2T8TD6M,sl,sierra-leone
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1yPQVFwbQ4edUfBhgAbL2o9DAOljZigTF,1KiMws3gla6Jc82CiGqgOXYyEg0Rvl8AefWznF6V6IKc,td,chad
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1cyrCvMKVRHJtpQtcbTpoboJc9iNZ-oHy,1FZ3WqAlNpBNFv7zWOTFqUC45FXE_zqLvYQzMP3RCiL0,ug,uganda
-1NS-tHBXYAGnFLS1VGhVFc7drQ59zD9h5d3sMTMOLMNY,1y_KMNV-wHDAcZgYnJyx-K7O9zZOuzrr7,1nhiu45GKNEc1kTwoZuG6KNKDMKVvFcn5Rg78WTRAKmU,ye,yemen
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1ztOfGaQT3WDrq-BOjT0x5VErzgrWQ0Ku,1Ck11zLFVP6iJZFAR0_Xsq0UaeEJrmFl7ysbFX9mGu7c,ae,united-arab-emirates
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1HpIjYaH_iMCRQD1jP159VGz-2NL4nB5p,1EqAi59wjE1v-bYX3cC1qdl6zkThpWJ8YcvSPUC-RGHc,bd,bangladesh
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1j8KgLnpjlnLy6bN4ozkwnBpkHUI6i3si,1wBmSuTkoEhosDzfHtyvZqd9SKez-sWoPoJ9oPonWsSo,bf,burkina-faso
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1mjTLMZ1la3zyiVQxLZ56sW497Sp8Lh5m,1c0O2XlwSpTAtB0AdhkkdgevWbsBUxvsmsETUwPPVIlk,bh,bahrain
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1qZoQciglG1DOeEa3hh5iUvF7q4_bKOQl,1cZVy2PUAzeq2xOoLRLwL9z9mqbry32zv_XY7sjEih2c,eg,egypt
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1eZHw6k7xM7Z_ZNMnF0Wc5cjJuVyuOop3,1CKkNsXwRdwXDiOldwT-6baw9DayXA2Vsn4ttpwP9SuM,jo,jordan
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1vnWgeTkq5TUyF7555F4renMJnl4WiFNy,1Y6-9-9kai-YyK1pXvcv_W6fqUn9lORltUhuFc2YUu1I,kw,kuwait
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,13XnZOF7U0uwL3EP_QpdTVd1FUh3A3cwi,1aGbMvFHzGn9ZlKKcFhiQ2c9egsoGDH11QBgyqmhS-IM,lr,liberia
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1fnTq_ONVxzVBbCyQy_-s6ngmadA6st70,16962grIJlisFbh2Zp9kBAhv6jVnZz6bHgb6RGBUHd3o,ma,morocco
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1syUOihIFKzk6QsLXd7XNUZIwzZZfAqEH,1UcgoJ_ytS-WSWl2_5OuV9h92wSCBWRFBoDtr4Ztqt14,ml,mali
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1dU24WM8MAnqApFcBoYCiKPzPevebal6_,1vwb7ENaOeVRJIc5iCDBbF8K0Oql4SscENmLEdUT77Hg,mm,myanmar
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,17Zqej6mrCT3BBBKcKj7949qHyRCa-9SJ,1cUtCEUuZRMqcxlRqFyoEM9eAdiDdWy2DUocroYivCx4,mr,mauritania
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1nVWV5_1kGDwyWJ3PPqExKfchs3sAlEuh,168KuHwUr9565zWaQVZ5au3qtGOb-qyJx_WOwNzqt_Eo,mx,mexico
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1qlHquI9EDz2lteBcjz_MheNLspg3mp_q,1_Pj5BryFXUPQPmMigII8G2HBUrpsnkK5V-Zu_9LCdGw,ne,niger
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1boFMPurqyxbfYBvfisRiROmzr8TuPI8j,1f3W3qJATCzVjZGw239Wy3D25THs8ThnvoC24aUFaGZQ,ng,nigeria
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1nMXXtFwJ3TqeynpKSW11uYAzihSMV8So,1Uc5eZswLB6mrwQLhd_OYQm7v7ThH99N0eb7RbTtD5iY,np,nepal
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,19o8a1zwxUEYFxvZkqs2AwCyIm0oe_CPF,1h1a0S5aVv9Z3wucgKsYXmg5Z_CWzsKfjJSfJFcXxPSY,ph,philippines
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1x4LjK_UWdxZm5EyNCupj7ikv7E-WMrkd,1UGOxjmJdJ9Dzj8cX3mZkgXAzT_ap_EMD2OqLjzDeGeE,qa,qatar
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1wSzKP9bsXB9w4U8frj4Y5kHrfV3C6Vi8,1QAgVpj0bf_A0HGFzHgwxBbZqgIFurfH4h7u1MnfKzJc,rw,rwanda
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1FLls5iHQD71Omy4VEzXYQ4HacMubzg8v,1a9XRXK5rG4_n0Afw7tIDkIbAmdydqKcU8J8zx5pLnVU,sa,saudi-arabia
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1tNS4yJYlh265zDy9rQnjAZSqSmBZxrRh,11dEjFSe56YdmJfVeKhRZpQKSgRb6mfM1DWKoNFxYg9Y,sd,sudan
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1l3NE8P8Xi-1qGwqZcdVdvV3Hn1h4Bwjv,1YxRrB39ItO_kEPTrMQ9FJlvMEp1Fjby0vchHiwW3C_I,sl,sierra-leone
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1yPQVFwbQ4edUfBhgAbL2o9DAOljZigTF,15cnbBqIlp4LzEXrs2z2L4_RTnY5e1GMrGV150JV615Q,td,chad
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1cyrCvMKVRHJtpQtcbTpoboJc9iNZ-oHy,1WlN4Hbv3JKE76hnNYkr80HU9oNJwjjOnj9nt7mm9ddw,ug,uganda
+1dkGS6Ocyc2YYQ5IopEdjz38t9kaT34XfIAab0iteGoY,1Ii31JX8y2InKt-FnHK-6kaqVK41XBOzY,1r62axKA5xgvJAiSiHrKgHZSATwSkKB-K15fdmLbn3zo,ye,yemen