Remove WRITE_TRUNCATE option when writing to BQ.

googlegenomics · Feb 5, 2020 · 8c632c2 · 8c632c2
1 parent 1b25caf
commit 8c632c2
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 12 deletions.
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -47,6 +47,8 @@ RUN apt-get update && apt-get install -y \
     zlib1g-dev
 
 # Install dependencies.
+
+
 RUN python -m pip install --upgrade pip && \
     python -m pip install --upgrade virtualenv && \
     virtualenv /opt/gcp_variant_transforms/venv && \

diff --git a/gcp_variant_transforms/transforms/sample_info_to_bigquery.py b/gcp_variant_transforms/transforms/sample_info_to_bigquery.py
@@ -73,14 +73,11 @@ def __init__(self, output_table_prefix, sample_name_encoding, append=False):
   def expand(self, pcoll):
     return (pcoll
             | 'ConvertSampleInfoToBigQueryTableRow' >> beam.ParDo(
-                ConvertSampleInfoToRow(self._sample_name_encoding))
-            | 'WriteSampleInfoToBigQuery' >> beam.io.Write(beam.io.BigQuerySink(
+                ConvertSampleInfoToRow(self.sample_name_encoding))
+            | 'WriteSampleInfoToBigQuery' >> beam.io.WriteToBigQuery(
                 self._output_table,
                 schema=self._schema,
                 create_disposition=(
                     beam.io.BigQueryDisposition.CREATE_IF_NEEDED),
-                write_disposition=(
-                    beam.io.BigQueryDisposition.WRITE_APPEND
-                    if self._append
-                    else beam.io.BigQueryDisposition.WRITE_TRUNCATE),
+                write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND,
                 method=beam.io.WriteToBigQuery.Method.FILE_LOADS))
diff --git a/gcp_variant_transforms/transforms/variant_to_bigquery.py b/gcp_variant_transforms/transforms/variant_to_bigquery.py
@@ -115,8 +115,5 @@ def expand(self, pcoll):
                 schema=self._schema,
                 create_disposition=(
                     beam.io.BigQueryDisposition.CREATE_IF_NEEDED),
-                write_disposition=(
-                    beam.io.BigQueryDisposition.WRITE_APPEND
-                    if self._append
-                    else beam.io.BigQueryDisposition.WRITE_TRUNCATE),
+                write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND,
                 method=beam.io.WriteToBigQuery.Method.FILE_LOADS))
diff --git a/gcp_variant_transforms/vcf_to_bq.py b/gcp_variant_transforms/vcf_to_bq.py
@@ -483,8 +483,6 @@ def run(argv=None):
     num_shards = 1
 
   if known_args.output_table:
-<<<<<<< HEAD
-<<<<<<< HEAD
     schema_file = tempfile.mkstemp(prefix=known_args.output_table,
                                    suffix=_BQ_SCHEMA_FILE_SUFFIX)[1]
     schema = (