Skip to content

Commit

Permalink
[main] --load-only
Browse files Browse the repository at this point in the history
Obcas uz mam CSV hotovy, ale z nejakyho duvodu neprosel load nebo menit
SQL schema a chci to nahrat znova. Pomoci load-only se vyhnem promazani
CSV adresare a jde se rovnou nahravat do db.
  • Loading branch information
kokes committed Jul 11, 2022
1 parent c881170 commit 883d3ce
Showing 1 changed file with 13 additions and 6 deletions.
19 changes: 13 additions & 6 deletions main.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,11 @@
action="store_true",
help="před importem smaz tabulky (dobre pri zmene schematu)",
)
parser.add_argument(
"--load-only",
action="store_true",
help="nestahuj data, jen nahraj existující CSV do databáze",
)
parser.add_argument(
"--partial",
action="store_true",
Expand All @@ -37,6 +42,11 @@
"specifikuj bud --all, nebo specificke datasety, ne oboji"
)

if args.load_only and not args.connstring:
raise argparse.ArgumentError(
"při --load-only je třeba specifikovat --connstring"
)

base_outdir = "csv"
os.makedirs(base_outdir, exist_ok=True)

Expand Down Expand Up @@ -70,10 +80,6 @@
module_names = args.modules
modules = {}
schemas = {}
# TODO: copy commands (or perhaps after `module(outdir)`? List that dir,
# open csv, do an executemany)
# also make sure we TRUNCATE each table before we insert into it (because
# we may have skipped the CREATE part)

for module in module_names:
modules[module] = import_module(f"data.{module}.main").main
Expand All @@ -85,11 +91,12 @@
print("=" * len(module_name))

outdir = os.path.join(base_outdir, module_name)
if os.path.isdir(outdir):
if os.path.isdir(outdir) and not args.load_only:
shutil.rmtree(outdir)
os.makedirs(outdir, exist_ok=True)

module(outdir, partial=args.partial)
if not args.load_only:
module(outdir, partial=args.partial)

if engine:
table_loads = defaultdict(list)
Expand Down

0 comments on commit 883d3ce

Please sign in to comment.