Skip to content

Commit

Permalink
refactor: unzip in a temporary folder instead (#834)
Browse files Browse the repository at this point in the history
## 🔧 Problem

We cannot run two imports in parallel because the csv files are unzipped
in the same directory.
 
## 🍰 Solution

Unzip in a temporary folder instead

## 🏝️ How to test

Extract two clones of ecobalyse in two different folders, and launch the
make process at the same time. Check both work.
  • Loading branch information
ccomb authored Nov 20, 2024
1 parent ca8a987 commit 8d01b8f
Showing 1 changed file with 25 additions and 25 deletions.
50 changes: 25 additions & 25 deletions data/common/import_.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import functools
import json
import os
import re
import sys
from os.path import dirname
import tempfile
from os.path import basename, join, splitext
from subprocess import call
from zipfile import ZipFile

Expand Down Expand Up @@ -186,29 +186,29 @@ def import_simapro_csv(
"""
print(f"### Importing {datapath}...")
# unzip
with ZipFile(datapath) as zf:
print("### Extracting the zip file...")
zf.extractall(path=dirname(datapath))
unzipped = datapath[0:-4]

if "AGB" in datapath:
print("### Patching Agribalyse...")
# `yield` is used as a variable in some Simapro parameters. bw2parameters cannot handle it:
# (sed is faster than Python)
call("sed -i 's/yield/Yield_/g' " + unzipped, shell=True)
# Fix some errors in Agribalyse:
call("sed -i 's/01\\/03\\/2005/1\\/3\\/5/g' " + unzipped, shell=True)
call("sed -i 's/\"0;001172\"/0,001172/' " + unzipped, shell=True)

print(f"### Importing into {dbname}...")
# Do the import
database = bw2io.importers.simapro_csv.SimaProCSVImporter(
unzipped, dbname, normalize_biosphere=True
)
if source:
for ds in database:
ds["source"] = source
os.unlink(unzipped)
with tempfile.TemporaryDirectory() as tempdir:
with ZipFile(datapath) as zf:
print(f"### Extracting the zip file in {tempdir}...")
zf.extractall(path=tempdir)
unzipped, _ = splitext(join(tempdir, basename(datapath)))

if "AGB" in datapath:
print("### Patching Agribalyse...")
# `yield` is used as a variable in some Simapro parameters. bw2parameters cannot handle it:
# (sed is faster than Python)
call("sed -i 's/yield/Yield_/g' " + unzipped, shell=True)
# Fix some errors in Agribalyse:
call("sed -i 's/01\\/03\\/2005/1\\/3\\/5/g' " + unzipped, shell=True)
call("sed -i 's/\"0;001172\"/0,001172/' " + unzipped, shell=True)

print(f"### Importing into {dbname}...")
# Do the import
database = bw2io.importers.simapro_csv.SimaProCSVImporter(
unzipped, dbname, normalize_biosphere=True
)
if source:
for ds in database:
ds["source"] = source

print("### Applying migrations...")
# Apply provided migrations
Expand Down

0 comments on commit 8d01b8f

Please sign in to comment.