diff --git a/.github/workflows/branch-deploy.yml b/.github/workflows/branch-deploy.yml index d344018..dcf0149 100644 --- a/.github/workflows/branch-deploy.yml +++ b/.github/workflows/branch-deploy.yml @@ -28,7 +28,11 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - run: pip install dokkusd - - run: python -m dokkusd.cli deploy --appname ${{ secrets.DOKKUSD_BRANCH_APP_NAME_PREFIX }}-${GITHUB_REF##*/} + - uses: oNaiPs/secrets-to-env-action@v1 + with: + secrets: ${{ toJSON(secrets) }} + prefix: ALLSECRETS_ + - run: python -m dokkusd.cli deploy --appname ${{ secrets.DOKKUSD_BRANCH_APP_NAME_PREFIX }}-${GITHUB_REF##*/} --environmentvariablesprefixedby ALLSECRETS_DOKKUSD_BRANCH_ENVIRONMENT_VARIABLE_ env: DOKKUSD_REMOTE_HOST: ${{ secrets.DOKKUSD_BRANCH_REMOTE_HOST }} DOKKUSD_REMOTE_PORT: ${{ secrets.DOKKUSD_BRANCH_REMOTE_PORT }} diff --git a/.github/workflows/live-deploy.yml b/.github/workflows/live-deploy.yml index a7e647d..96e76ec 100644 --- a/.github/workflows/live-deploy.yml +++ b/.github/workflows/live-deploy.yml @@ -28,7 +28,11 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - run: pip install dokkusd - - run: python -m dokkusd.cli deploy --appname ${{ secrets.DOKKUSD_LIVE_APP_NAME }} + - uses: oNaiPs/secrets-to-env-action@v1 + with: + secrets: ${{ toJSON(secrets) }} + prefix: ALLSECRETS_ + - run: python -m dokkusd.cli deploy --appname ${{ secrets.DOKKUSD_LIVE_APP_NAME }} --environmentvariablesprefixedby ALLSECRETS_DOKKUSD_LIVE_ENVIRONMENT_VARIABLE_ env: DOKKUSD_REMOTE_HOST: ${{ secrets.DOKKUSD_LIVE_REMOTE_HOST }} DOKKUSD_REMOTE_PORT: ${{ secrets.DOKKUSD_LIVE_REMOTE_PORT }} diff --git a/cove_ofds/forms.py b/cove_ofds/forms.py index 7eafc18..049f1e4 100644 --- a/cove_ofds/forms.py +++ b/cove_ofds/forms.py @@ -4,6 +4,7 @@ class NewGeoJSONUploadForm(forms.Form): nodes_file_upload = forms.FileField( + label="Select GeoJSON Nodes file", widget=forms.FileInput( attrs={ "accept": ",".join( @@ -11,9 +12,10 @@ class NewGeoJSONUploadForm(forms.Form): + settings.ALLOWED_GEOJSON_EXTENSIONS ) } - ) + ), ) spans_file_upload = forms.FileField( + label="Select GeoJSON Spans file", widget=forms.FileInput( attrs={ "accept": ",".join( @@ -21,5 +23,5 @@ class NewGeoJSONUploadForm(forms.Form): + settings.ALLOWED_GEOJSON_EXTENSIONS ) } - ) + ), ) diff --git a/cove_ofds/process.py b/cove_ofds/process.py index 509fff2..353f076 100644 --- a/cove_ofds/process.py +++ b/cove_ofds/process.py @@ -1,5 +1,6 @@ import json import os.path +import zipfile import flattentool from libcoveofds.additionalfields import AdditionalFields @@ -97,6 +98,55 @@ def get_context(self): return context +class ConvertCSVsIntoJSON(ProcessDataTask): + """If User uploaded CSVs, convert to our primary format, JSON.""" + + def process(self, process_data: dict) -> dict: + if self.supplied_data.format != "csvs": + return process_data + + # check already done + # TODO + + output_dir = os.path.join(self.supplied_data.data_dir(), "unflatten") + + os.makedirs(output_dir, exist_ok=True) + + unflatten_kwargs = { + "output_name": os.path.join(output_dir, "unflattened.json"), + "root_list_path": "networks", + "input_format": "csv", + } + + flattentool.unflatten(self.supplied_data.upload_dir(), **unflatten_kwargs) + + process_data["json_data_filename"] = os.path.join( + self.supplied_data.data_dir(), "unflatten", "unflattened.json" + ) + + return process_data + + def get_context(self): + context = {} + # original format + if self.supplied_data.format == "csvs": + context["original_format"] = "csvs" + # Download data + filename = os.path.join( + self.supplied_data.data_dir(), "unflatten", "unflattened.json" + ) + if os.path.exists(filename): + context["can_download_json"] = True + context["download_json_url"] = os.path.join( + self.supplied_data.data_url(), "unflatten", "unflattened.json" + ) + context["download_json_size"] = os.stat(filename).st_size + else: + context["can_download_json"] = False + # Return + return context + + class ConvertGeoJSONIntoJSON(ProcessDataTask): """If User uploaded GeoJSON, convert to our primary format, JSON.""" @@ -230,6 +280,12 @@ def get_context(self): class ConvertJSONIntoSpreadsheets(ProcessDataTask): """Convert primary format (JSON) to spreadsheets""" + def __init__(self, supplied_data): + super().__init__(supplied_data) + self.csvs_zip_filename = os.path.join( + self.supplied_data.data_dir(), "flatten", "flattened.csvs.zip" + ) + def process(self, process_data: dict) -> dict: # TODO don't run if already done @@ -243,6 +299,12 @@ def process(self, process_data: dict) -> dict: flattentool.flatten(process_data["json_data_filename"], **flatten_kwargs) + # Make Zip file of all CSV files + with zipfile.ZipFile(self.csvs_zip_filename, "w") as out_zip: + for f in os.listdir(output_dir): + if os.path.isfile(os.path.join(output_dir, f)) and f.endswith(".csv"): + out_zip.write(os.path.join(output_dir, f), arcname=f) + return process_data def get_context(self): @@ -271,6 +333,15 @@ def get_context(self): context["download_ods_size"] = os.stat(ods_filename).st_size else: context["can_download_ods"] = False + # CSVs + if os.path.exists(self.csvs_zip_filename): + context["can_download_csvs_zip"] = True + context["download_csvs_zip_url"] = os.path.join( + self.supplied_data.data_url(), "flatten", "flattened.csvs.zip" + ) + context["download_csvs_zip_size"] = os.stat(ods_filename).st_size + else: + context["can_download_csvs_zip"] = False # done! return context @@ -291,13 +362,78 @@ def process(self, process_data: dict) -> dict: schema = OFDSSchema() worker = PythonValidate(schema) - context = {"additional_checks": worker.validate(data)} + + # has_links_with_external_node_data and has_links_with_external_span_data are shown in a different bit of UI. + # Set variables and move out of additional_checks + context["has_links_with_external_node_data"] = ( + True + if [ + r + for r in context["additional_checks"] + if r["type"] == "has_links_with_external_node_data" + ] + else False + ) + context["has_links_with_external_span_data"] = ( + True + if [ + r + for r in context["additional_checks"] + if r["type"] == "has_links_with_external_span_data" + ] + else False + ) + context["additional_checks"] = [ + r + for r in context["additional_checks"] + if ( + r["type"] != "has_links_with_external_node_data" + and r["type"] != "has_links_with_external_span_data" + ) + ] + + # Count and group what's left context["additional_checks_count"] = len(context["additional_checks"]) context["additional_checks"] = group_data_list_by( context["additional_checks"], lambda i: i["type"] ) + # The library returns *_name_does_not_match and *_reference_name_set_but_not_in_original as different types, + # but in this UI we don't care - we just want to show them as one section. + # So join the 2 types of errors into 1 list. + for f1, f2 in [ + ( + "node_phase_reference_name_does_not_match", + "node_phase_reference_name_set_but_not_in_original", + ), + ( + "span_phase_reference_name_does_not_match", + "span_phase_reference_name_set_but_not_in_original", + ), + ( + "contract_related_phase_reference_name_does_not_match", + "contract_related_phase_reference_name_set_but_not_in_original", + ), + ( + "node_organisation_reference_name_does_not_match", + "node_organisation_reference_name_set_but_not_in_original", + ), + ( + "span_organisation_reference_name_does_not_match", + "span_organisation_reference_name_set_but_not_in_original", + ), + ( + "phase_organisation_reference_name_does_not_match", + "phase_organisation_reference_name_set_but_not_in_original", + ), + ]: + new_list = context["additional_checks"].get(f1, []) + context[ + "additional_checks" + ].get(f2, []) + if new_list: + context["additional_checks"][f1] = new_list + with open(self.data_filename, "w") as fp: json.dump(context, fp, indent=4) diff --git a/cove_ofds/templates/cove_ofds/additional_checks_table.html b/cove_ofds/templates/cove_ofds/additional_checks_table.html index ed406cc..94a502a 100644 --- a/cove_ofds/templates/cove_ofds/additional_checks_table.html +++ b/cove_ofds/templates/cove_ofds/additional_checks_table.html @@ -4,6 +4,12 @@ {% if 'span_start_node_not_found' in additional_checks or 'span_end_node_not_found' in additional_checks %}

{% trans 'Node references' %}

{% trans 'Your data contains spans with node references that cannot be resolved. `Span.start` and `Span.end` must match the `.id` of exactly one node in the `/nodes` array.' %}

+ {% if 'span_start_node_not_found' in additional_checks %} +

{% trans 'Schema Documentation' %}

+ {% endif %} + {% if 'span_end_node_not_found' in additional_checks %} +

{% trans 'Schema Documentation' %}

+ {% endif %} @@ -47,6 +53,7 @@

{% trans 'Node references' %}

{% if 'node_location_type_incorrect' in additional_checks %}

{% trans 'Node location type' %}

{% trans 'Your data contains nodes with incorrect location types. `/nodes/location/type` must be `Point`.' %}

+

{% trans 'Schema Documentation' %}

@@ -73,6 +80,7 @@

{% trans 'Node location type' %}

{% if 'node_location_coordinates_incorrect' in additional_checks %}

{% trans 'Node location coordinates' %}

{% trans 'Your data contains nodes with incorrectly formatted location coordinates. `/nodes/location/coordinates` must be a single position, i.e. an array of numbers.' %}

+

{% trans 'Schema Documentation' %}

@@ -99,6 +107,7 @@

{% trans 'Node location coordinates' %}

{% if 'span_route_type_incorrect' in additional_checks %}

{% trans 'Span route geometry' %}

{% trans 'Your data contains spans with incorrect route types. `/spans/route/type` must be `LineString`.' %}

+

{% trans 'Schema Documentation' %}

@@ -125,6 +134,7 @@

{% trans 'Span route geometry' %}

{% if 'span_route_coordinates_incorrect' in additional_checks %}

{% trans 'Span route coordinates' %}

{% trans 'Your data contains spans with incorrectly formatted route coordinates. `/spans/route/coordinates` must be an array of positions, i.e. an array of arrays of numbers.' %}

+

{% trans 'Schema Documentation' %}

@@ -151,6 +161,7 @@

{% trans 'Span route coordinates' %}

{% if 'node_phase_reference_id_not_found' in additional_checks or 'span_phase_reference_id_not_found' in additional_checks or 'contract_related_phase_reference_id_not_found' in additional_checks %}

{% trans 'Phase references' %}

{% trans 'Your data contains phase references that cannot be resolved. The `.id` of each phase reference must match the `.id` of exactly one phase in the `/phases` array.' %}

+

{% trans 'Schema Documentation' %}

{% if 'node_phase_reference_id_not_found' in additional_checks %}
@@ -225,6 +236,7 @@

{% trans 'Phase references' %}

{% if 'node_phase_reference_name_does_not_match' in additional_checks or 'span_phase_reference_name_does_not_match' in additional_checks or 'contract_related_phase_reference_name_does_not_match' in additional_checks %}

{% trans 'Phase names' %}

{% trans 'Your data contains phase references with inconsistent names. The `.name` of each phase reference must match the `.name` of the phase it references.' %}

+

{% trans 'Schema Documentation' %}

{% if 'node_phase_reference_name_does_not_match' in additional_checks %}
@@ -293,89 +305,11 @@

{% trans 'Phase names' %}

{% endif %} {% endif %} -{% if 'node_phase_reference_name_set_but_not_in_original' in additional_checks %} -

{% trans 'TODO node_phase_reference_name_set_but_not_in_original' %}

-

{% trans 'TODO' %}

-
- - - - - - - - {% for additional_check in additional_checks.node_phase_reference_name_set_but_not_in_original %} - - - - - {% endfor %} - -
{% trans 'Node Id' %}{% trans 'Network ID' %}
- {{ additional_check.node_id }} - - {{ additional_check.network_id }} -
-{% endif %} - - - - -{% if 'span_phase_reference_name_set_but_not_in_original' in additional_checks %} -

{% trans 'TODO span_phase_reference_name_set_but_not_in_original' %}

-

{% trans 'TODO' %}

- - - - - - - - - {% for additional_check in additional_checks.span_phase_reference_name_set_but_not_in_original %} - - - - - {% endfor %} - -
{% trans 'Span Id' %}{% trans 'Network ID' %}
- {{ additional_check.span_id }} - - {{ additional_check.network_id }} -
-{% endif %} - - -{% if 'contract_related_phase_reference_name_set_but_not_in_original' in additional_checks %} -

{% trans 'TODO span_phase_reference_name_set_but_not_in_original' %}

-

{% trans 'TODO' %}

- - - - - - - - - {% for additional_check in additional_checks.contract_related_phase_reference_name_set_but_not_in_original %} - - - - - {% endfor %} - -
{% trans 'Contract Id' %}{% trans 'Network ID' %}
- {{ additional_check.contract_id }} - - {{ additional_check.network_id }} -
-{% endif %} - {% if 'node_organisation_reference_id_not_found' in additional_checks or 'node_organisation_reference_id_not_found' in additional_checks or 'phase_organisation_reference_id_not_found' in additional_checks %}

{% trans 'Organisation references' %}

{% trans 'Your data contains organisation references that cannot be resolved. The `.id` of each organisation reference must match the `.id` of exactly one organisation in the `/organisations` array.' %}

+

{% trans 'Schema Documentation' %}

{% if 'node_organisation_reference_id_not_found' in additional_checks %} @@ -457,6 +391,7 @@

{% trans 'Organisation references' %}

{% if 'node_organisation_reference_name_does_not_match' in additional_checks %}

{% trans 'Organisation names' %}

{% trans 'Your data contains organisation references with inconsistent names. The `.name` of each organisation reference must match the `.name` of the organisation it references.' %}

+

{% trans 'Schema Documentation' %}

{% if 'node_organisation_reference_name_does_not_match' in additional_checks %}
@@ -534,97 +469,10 @@

{% trans 'Organisation names' %}

{% endif %} -{% if 'node_organisation_reference_name_set_but_not_in_original' in additional_checks %} -

{% trans 'TODO node_organisation_reference_name_set_but_not_in_original' %}

-

{% trans 'TODO ' %}

-
- - - - - - - - - {% for additional_check in additional_checks.node_organisation_reference_name_set_but_not_in_original %} - - - - - - {% endfor %} - -
{% trans 'Field' %}{% trans 'Node Id' %}{% trans 'Network ID' %}
- {{ additional_check.field }} - - {{ additional_check.node_id }} - - {{ additional_check.network_id }} -
-{% endif %} - - -{% if 'span_organisation_reference_name_set_but_not_in_original' in additional_checks %} -

{% trans 'TODO span_organisation_reference_name_set_but_not_in_original' %}

-

{% trans 'TODO ' %}

- - - - - - - - - - {% for additional_check in additional_checks.span_organisation_reference_name_set_but_not_in_original %} - - - - - - {% endfor %} - -
{% trans 'Field' %}{% trans 'Span Id' %}{% trans 'Network ID' %}
- {{ additional_check.field }} - - {{ additional_check.span_id }} - - {{ additional_check.network_id }} -
-{% endif %} - - - - -{% if 'phase_organisation_reference_name_set_but_not_in_original' in additional_checks %} -

{% trans 'TODO phase_organisation_reference_name_set_but_not_in_original' %}

-

{% trans 'TODO ' %}

- - - - - - - - - {% for additional_check in additional_checks.phase_organisation_reference_name_set_but_not_in_original %} - - - - - {% endfor %} - -
{% trans 'Phase Id' %}{% trans 'Network ID' %}
- {{ additional_check.phase_id }} - - {{ additional_check.network_id }} -
-{% endif %} - - {% if 'node_international_connections_country_not_set' in additional_checks %}

{% trans 'International connection countries' %}

{% trans 'Your data contains nodes with international connections that do not specify a country. `.country` must be set for each international connection in `/nodes/internationalConnections`.' %}

+

{% trans 'Schema Documentation' %}

@@ -648,20 +496,8 @@

{% trans 'International connection countries' %}

{% endif %} -{% if 'has_links_with_external_node_data' in additional_checks %} -

{% trans 'Links to nodes data' %}

-

{% trans 'Your data contains links to API endpoints or bulk files for nodes. The additional data available from the links has not been checked. You can check the data by embedding it in network package and submitting it to CoVE.' %}

-{% endif %} - - -{% if 'has_links_with_external_span_data' in additional_checks %} -

{% trans 'Links to spans data' %}

-

{% trans 'Your data contains links to API endpoints or bulk files for nodes. The additional data available from the links has not been checked. You can check the data by embedding it in network package and submitting it to CoVE.' %}

-{% endif %} - - {% if 'node_not_used_in_any_spans' in additional_checks %} -

{% trans 'Orphaned nodes' %}

+

{% trans 'Unreferenced nodes' %}

{% trans 'Your data contains nodes that are not referenced by any spans.' %}

diff --git a/cove_ofds/templates/cove_ofds/base.html b/cove_ofds/templates/cove_ofds/base.html index f29d422..bc832da 100644 --- a/cove_ofds/templates/cove_ofds/base.html +++ b/cove_ofds/templates/cove_ofds/base.html @@ -24,16 +24,16 @@
-

Open Fibre Data Standard (OFDS)

+

Open Fibre Data Standard
- {% block h1 %}

{% blocktrans %} CoVE{% endblocktrans %}

beta

{% endblock %} + {% block h1 %}

{% blocktrans %} CoVE{% endblocktrans %}

Convert, Validate, Explore

{% endblock %}
-

{% blocktrans %} OFDS Documentation {% endblocktrans %}

 +

{% blocktrans %} Standard Documentation {% endblocktrans %}
@@ -56,7 +56,7 @@

Open Fibre Data Standard (OFDS)

{% endblock %} {% block link %} -
  • {% trans "Open Fibre Data Standard" %}
    • +
  • {% trans "Open Fibre Data Standard Documentation" %}
    • {% endblock %} {% block bottomcontent1 %} diff --git a/cove_ofds/templates/cove_ofds/explore.html b/cove_ofds/templates/cove_ofds/explore.html index 53cf444..82d2206 100644 --- a/cove_ofds/templates/cove_ofds/explore.html +++ b/cove_ofds/templates/cove_ofds/explore.html @@ -8,85 +8,160 @@ {% block explore_content %}
    -
    +

    {% trans 'Schema Version' %}

    -

    {% trans 'Schema Version Used' %}: {{ schema_version_used }}

    +

    {% trans 'Your data was checked against schema version' %}: {{ schema_version_used }}

    -
    -
    -
    - {% trans 'Download Data' %} + {% if has_links_with_external_node_data or has_links_with_external_span_data %} +
    +
    +
    +

    + {% trans 'Unchecked data' %}

    -
    - -

    - {% if original_format == 'json' %} - {% trans 'You uploaded data in JSON format.' %} - {% elif original_format == 'spreadsheet' %} - {% trans 'You uploaded data in a spreadsheet format.' %} - {% elif original_format == 'geojson' %} - {% trans 'You uploaded data in a GeoJSON format.' %} - {% endif %} -

    - -

    - {% trans 'You can download your original data:' %} -

    - - - - {% if can_download_geojson or can_download_xlsx or can_download_ods or can_download_json %} -

    {% trans 'We converted this to the following formats for you:' %}

    - - {% endif %} +
    +

    {% trans 'Your data contains links to API endpoints or bulk files. The additional data available from the links has not been checked. You can check the data by embedding it in network package and submitting it to CoVE. For more information, see ' %}/links.

    +
    +
    + {% endif %} + +
    + +
    +
    +

    + {% trans 'Download Data' %} +

    +
    +
    + +

    + For more information, see the publication format reference. +

    + +
    +
    + + +
    +
    +

    + {% trans 'JSON' %} {% if original_format == 'json' %}{% trans '(original)' %}{% endif %} +

    +
    +
    + {% if original_format == 'json' %} + + {% else %} + {% if can_download_json %} + + {% endif %} + {% endif %} +
    +
    + +
    +
    + +
    +
    +

    + {% trans 'GeoJSON' %} {% if original_format == 'geojson' %}{% trans '(original)' %}{% endif %} +

    +
    +
    + {% if original_format == 'geojson' %} + + {% else %} + {% if can_download_geojson %} + + {% endif %} + {% endif %} +
    +
    + +
    +
    + +
    +
    +

    + {% trans 'CSVs' %} {% if original_format == 'csvs' %}{% trans '(original)' %}{% endif %} +

    +
    +
    + {% if original_format == 'csvs' %} + + {% else %} + {% if can_download_csvs_zip %} + + {% endif %} + {% endif %} +
    +
    + +
    +
    +
    +
    +
    @@ -98,10 +173,11 @@

    - {% trans 'Validation Errors' %} + {% trans 'Structure and Format' %}

    +

    {% trans 'The structure and format of your data does not conform to the OFDS schema. You should check your mapping and data pipeline for errors. For more information, see the ' %}{% trans 'reference documentation' %}.

    {% include "cove_ofds/validation_table.html" %}
    @@ -109,11 +185,11 @@

    - {% trans 'Validation Errors' %} + {% trans 'Structure and Format' %}

    -

    {% trans 'There were no validation errors!' %}

    +

    {% trans 'The structure and format of your data conforms to the OFDS schema.' %}

    {% endif %} @@ -127,25 +203,32 @@

    {% if additional_fields_count %} -
    -
    +
    +

    {% trans 'Additional Fields' %}

    +

    {% trans 'Your data contains additional fields that are not part of the OFDS schema. You should:' %}

    +
      +
    • {% trans 'Check that additional fields are not the result of typos in field names or other errors in your mapping or data pipeline.' %}
      • +
    • {% trans 'Check whether the data in these fields could be provided by using a field in the OFDS schema.' %}
      • +
    • {% trans 'Document the structure, format and meaning of additional fields in your' %}{% trans 'data user guide' %}.
      • +
    +

    {% trans 'For more information, see ' %}{% trans 'how to add additional fields' %}.

    {% include "libcoveweb2/additional_fields_table.html" %}
    {% else %} -
    -
    +
    +

    {% trans 'Additional Fields' %}

    -

    {% trans 'There were no additional fields!' %}

    +

    {% trans 'Your data contains no additional fields. For more information, see ' %}{% trans 'how to add additional fields' %}.

    {% endif %} @@ -164,9 +247,10 @@

    {% if additional_checks_count %} +

    {% trans 'Your data failed the following additional checks. You should check your mapping and data pipeline for errors.' %}

    {% include "cove_ofds/additional_checks_table.html" %} {% else %} -

    {% trans 'All checks passed!' %}

    +

    {% trans 'Your data passed all additional checks.' %}

    {% endif %}
    @@ -190,19 +274,20 @@

    -
    -
    +

    - {% trans 'Map' %} + {% trans 'Visualisation' %}

    {% if can_download_geojson %} +

    {% trans 'The GeoJSON version of your data is visualised on the map below. You should check that nodes and spans appear in the correct location. If not, you may need to ' %}{% trans 'transform your coordinates to the correct coordinate reference system' %}.

    {% else %}
    -

    {% trans 'There are no GeoJSON files to visualise!' %}

    +

    {% trans 'Your data cannot be visualised because it is not available in GeoJSON format. For more information, see ' %}{% trans 'data conversion' %}.

    {% endif %}
    diff --git a/cove_ofds/templates/cove_ofds/index.html b/cove_ofds/templates/cove_ofds/index.html index 28ef289..2cc1f54 100644 --- a/cove_ofds/templates/cove_ofds/index.html +++ b/cove_ofds/templates/cove_ofds/index.html @@ -4,14 +4,137 @@ {% block content %} - - -
    - Upload GeoJSON + +

    + Use the form below to submit your data. You can submit data in either JSON, GeoJSON or CSV format. + For more information, see the publication format reference. +

    + + + + + +
    +
    + + +
    + +
    +
    +
    {% csrf_token %} + {% bootstrap_form forms.json.upload_form %} + {% buttons %} + + {% endbuttons %} + +
    +
    +
    + + +
    + +
    +
    +
    {% csrf_token %} + {% bootstrap_form forms.json.text_form %} + {% buttons %} + + {% endbuttons %} + +
    +
    +
    + + +
    +
    + + +
    + +
    +
    +
    {% csrf_token %} + {% bootstrap_form forms.geojson.upload_form %} + {% buttons %} + + {% endbuttons %} + +
    +
    +
    + +
    +
    + + +
    + +
    +
    +
    {% csrf_token %} + {% bootstrap_form forms.csvs.upload_form %} + {% buttons %} + + {% endbuttons %} + +
    +
    +
    + +
    + + {% endblock %} diff --git a/cove_ofds/templates/cove_ofds/new_geojson.html b/cove_ofds/templates/cove_ofds/new_geojson.html index d3a0513..04f08f9 100644 --- a/cove_ofds/templates/cove_ofds/new_geojson.html +++ b/cove_ofds/templates/cove_ofds/new_geojson.html @@ -5,16 +5,27 @@ {% block content %} +
    -
    - {% csrf_token %} - {% bootstrap_form forms.upload_form %} - {% buttons %} - - {% endbuttons %} - + +
    +
    +
    {% csrf_token %} + {% bootstrap_form forms.upload_form %} + {% buttons %} + + {% endbuttons %} + +
    +
    {% endblock %} diff --git a/cove_ofds/test_functional.py b/cove_ofds/test_functional.py index 033ecdb..ffe35a4 100644 --- a/cove_ofds/test_functional.py +++ b/cove_ofds/test_functional.py @@ -32,7 +32,7 @@ def server_url(request, live_server): ("link_text", "url"), [ ( - "Open Fibre Data Standard", + "Open Fibre Data Standard Documentation", "https://open-fibre-data-standard.readthedocs.io/en/latest/", ), ], diff --git a/cove_ofds/views.py b/cove_ofds/views.py index b03af1f..c3c6ec9 100644 --- a/cove_ofds/views.py +++ b/cove_ofds/views.py @@ -9,6 +9,7 @@ from cove_ofds.forms import NewGeoJSONUploadForm from cove_ofds.process import ( AdditionalFieldsChecksTask, + ConvertCSVsIntoJSON, ConvertGeoJSONIntoJSON, ConvertJSONIntoGeoJSON, ConvertJSONIntoSpreadsheets, @@ -18,7 +19,7 @@ WasJSONUploaded, ) from libcoveweb2.models import SuppliedData -from libcoveweb2.views import explore_data_context +from libcoveweb2.views import CSVS_FORM_CLASSES, JSON_FORM_CLASSES, explore_data_context logger = logging.getLogger(__name__) @@ -30,9 +31,29 @@ def default(self, obj): return json.JSONEncoder.default(self, obj) +GEOJSON_FORM_CLASSES = { + "upload_form": NewGeoJSONUploadForm, +} + + def index(request): - return render(request, "cove_ofds/index.html", {}) + forms = { + "json": { + form_name: form_class() + for form_name, form_class in JSON_FORM_CLASSES.items() + }, + "csvs": { + form_name: form_class() + for form_name, form_class in CSVS_FORM_CLASSES.items() + }, + "geojson": { + form_name: form_class() + for form_name, form_class in GEOJSON_FORM_CLASSES.items() + }, + } + + return render(request, "cove_ofds/index.html", {"forms": forms}) def new_geojson(request): @@ -50,13 +71,13 @@ def new_geojson(request): not request.FILES[field].content_type in settings.ALLOWED_GEOJSON_CONTENT_TYPES ): - form.add_error("file_upload", "This does not appear to be a JSON file") + form.add_error(field, "This does not appear to be a JSON file") if not [ e for e in settings.ALLOWED_GEOJSON_EXTENSIONS if str(request.FILES[field].name).lower().endswith(e) ]: - form.add_error("file_upload", "This does not appear to be a JSON file") + form.add_error(field, "This does not appear to be a JSON file") # Process if form.is_valid(): @@ -85,6 +106,7 @@ def explore_ofds(request, pk): # Make sure uploads are in primary format WasJSONUploaded(db_data), ConvertSpreadsheetIntoJSON(db_data), + ConvertCSVsIntoJSON(db_data), ConvertGeoJSONIntoJSON(db_data), # Convert into output formats ConvertJSONIntoGeoJSON(db_data), diff --git a/cove_project/settings.py b/cove_project/settings.py index e2556fb..146ed30 100644 --- a/cove_project/settings.py +++ b/cove_project/settings.py @@ -158,7 +158,16 @@ # https://github.com/OpenDataServices/cove/issues/1098 FILE_UPLOAD_PERMISSIONS = 0o644 +ALLOWED_JSON_CONTENT_TYPES = settings.ALLOWED_JSON_CONTENT_TYPES +ALLOWED_JSON_EXTENSIONS = settings.ALLOWED_JSON_EXTENSIONS + +ALLOWED_SPREADSHEET_CONTENT_TYPES = settings.ALLOWED_SPREADSHEET_CONTENT_TYPES +ALLOWED_SPREADSHEET_EXTENSIONS = settings.ALLOWED_SPREADSHEET_EXTENSIONS + +ALLOWED_CSV_CONTENT_TYPES = settings.ALLOWED_CSV_CONTENT_TYPES +ALLOWED_CSV_EXTENSIONS = settings.ALLOWED_CSV_EXTENSIONS + ALLOWED_GEOJSON_CONTENT_TYPES = settings.ALLOWED_JSON_CONTENT_TYPES + [ "application/geo+json" ] -ALLOWED_GEOJSON_EXTENSIONS = settings.ALLOWED_JSON_EXTENSIONS +ALLOWED_GEOJSON_EXTENSIONS = settings.ALLOWED_JSON_EXTENSIONS + [".geojson"] diff --git a/libcoveweb2/forms.py b/libcoveweb2/forms.py index 530f46d..3e9d846 100644 --- a/libcoveweb2/forms.py +++ b/libcoveweb2/forms.py @@ -1,30 +1,148 @@ from django import forms - -from libcoveweb2.settings import ( - ALLOWED_JSON_CONTENT_TYPES, - ALLOWED_JSON_EXTENSIONS, - ALLOWED_SPREADSHEET_CONTENT_TYPES, - ALLOWED_SPREADSHEET_EXTENSIONS, -) +from django.conf import settings class NewJSONUploadForm(forms.Form): file_upload = forms.FileField( widget=forms.FileInput( attrs={ - "accept": ",".join(ALLOWED_JSON_CONTENT_TYPES + ALLOWED_JSON_EXTENSIONS) + "accept": ",".join( + settings.ALLOWED_JSON_CONTENT_TYPES + + settings.ALLOWED_JSON_EXTENSIONS + ) } - ) + ), + label="Select JSON file", ) +class NewJSONTextForm(forms.Form): + paste = forms.CharField(label="Paste (JSON only)", widget=forms.Textarea) + + class NewSpreadsheetUploadForm(forms.Form): file_upload = forms.FileField( widget=forms.FileInput( attrs={ "accept": ",".join( - ALLOWED_SPREADSHEET_CONTENT_TYPES + ALLOWED_SPREADSHEET_EXTENSIONS + settings.ALLOWED_SPREADSHEET_CONTENT_TYPES + + settings.ALLOWED_SPREADSHEET_EXTENSIONS ) } ) ) + + +class NewCSVsUploadForm(forms.Form): + # I know it's hacky to copy and paste code like this but as this needs to be replaced by + # something that allows any number of uploads with no limits this will do for now + file_field_names = ["file_upload" + str(i) for i in range(0, 10)] + file_upload0 = forms.FileField( + label="Select CSV file", + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ), + ) + file_upload1 = forms.FileField( + label="Select CSV file", + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ), + required=False, + ) + file_upload2 = forms.FileField( + label="Select CSV file", + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ), + required=False, + ) + file_upload3 = forms.FileField( + label="Select CSV file", + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ), + required=False, + ) + file_upload4 = forms.FileField( + label="Select CSV file", + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ), + required=False, + ) + file_upload5 = forms.FileField( + label="Select CSV file", + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ), + required=False, + ) + file_upload6 = forms.FileField( + label="Select CSV file", + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ), + required=False, + ) + file_upload7 = forms.FileField( + label="Select CSV file", + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ), + required=False, + ) + file_upload8 = forms.FileField( + label="Select CSV file", + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ), + required=False, + ) + file_upload9 = forms.FileField( + label="Select CSV file", + widget=forms.FileInput( + attrs={ + "accept": ",".join( + settings.ALLOWED_CSV_CONTENT_TYPES + settings.ALLOWED_CSV_EXTENSIONS + ) + } + ), + required=False, + ) diff --git a/libcoveweb2/models.py b/libcoveweb2/models.py index 4e9cc76..b7fd2e7 100644 --- a/libcoveweb2/models.py +++ b/libcoveweb2/models.py @@ -44,6 +44,29 @@ def save_file(self, f, meta={}): for chunk in f.chunks(): destination.write(chunk) + def save_file_contents( + self, + filename: str, + contents: str, + content_type: str, + charset: str = None, + meta: dict = {}, + ): + + os.makedirs(self.upload_dir(), exist_ok=True) + + supplied_data_file = SuppliedDataFile() + supplied_data_file.supplied_data = self + supplied_data_file.filename = filename + supplied_data_file.size = len(contents) + supplied_data_file.content_type = content_type + supplied_data_file.charset = charset + supplied_data_file.meta = meta + supplied_data_file.save() + + with open(supplied_data_file.upload_dir_and_filename(), "w") as destination: + destination.write(contents) + class SuppliedDataFile(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) diff --git a/libcoveweb2/settings.py b/libcoveweb2/settings.py index bfc3e7a..17969cb 100644 --- a/libcoveweb2/settings.py +++ b/libcoveweb2/settings.py @@ -182,3 +182,6 @@ "application/vnd.oasis.opendocument.spreadsheet", ] ALLOWED_SPREADSHEET_EXTENSIONS = [".ods", ".xlsx"] + +ALLOWED_CSV_CONTENT_TYPES = ["text/csv"] +ALLOWED_CSV_EXTENSIONS = [".csv"] diff --git a/libcoveweb2/static/dataexplore/css/style.css b/libcoveweb2/static/dataexplore/css/style.css new file mode 100644 index 0000000..e702171 --- /dev/null +++ b/libcoveweb2/static/dataexplore/css/style.css @@ -0,0 +1,156 @@ +.page-header a { + text-decoration:none; +} + +.page-header a.underline:hover { + text-decoration:underline; +} + +.page-header a.btn { + float:right +} + +h1.application-name { + margin-top: 0; +} + +.title360 { + font-size: 1.4em; + margin-top: 32px; +} + +.language-select { + margin: 1px 0; +} + +#accordion .panel-heading { + cursor: pointer; +} + +#more-information .panel-heading { + cursor: pointer; +} + +.heading-in-panel { + margin-top:0px; +} + +.pointer { + cursor: pointer; +} + +.panel-title a { + text-decoration:none; +} + +.panel-title .glyphicon { + padding-right:15px; +} + +#footer, .above-footer { + padding-top: 9px; + margin: 20px 0 40px; + border-top: 1px solid #eee; +} + +show-open-if-noscript { + display:block; +} + +#footer ul { + padding-left: 0; +} + +#footer ul li { + list-style-type: none; + +} + +.thumbnail { + background-color: #f4f8fb; + text-align: center; + } + +.copy-button { + padding: 3px; + position: relative; + top: -2px; + right: -5px; + color: grey +} + +.copy-div { + margin-bottom: 5px; + margin-top: -3px +} + +.copy-span { + border: 1px solid #DDD; + border-radius: 4px; + padding: 4px; +} + +a:hover { + cursor:pointer; +} + +.left-space { + margin-left:10px +} + +.indent { + padding-left:20px +} + +.success-button { + padding-bottom: 20px; +} + +.spinner { + margin-bottom: 20px; +} + +.explanation { + font-size: 1.2em; + margin-bottom: 0px; +} + +a.anchor::before { + display: block; + content: " "; + margin-top: -30px; + height: 30px; + visibility: hidden; +} + +.message { + margin-left:30px; + margin-bottom: 10px; +} +.message span { + margin-left: -30px; + padding-right:15px; +} +.key-facts ul { + padding-left: 0; + list-style: none; +} +.key-facts ul li { + margin: 4px 0; +} + +.key-facts .glyphicon-flag { + color: #a94442; +} + +.no-bottom-margin { + margin-bottom: 0px; +} + +table { + table-layout: fixed; +} + +table td { + overflow-wrap: break-word; +} diff --git a/libcoveweb2/static/dataexplore/fonts/bootstrap/glyphicons-halflings-regular.eot b/libcoveweb2/static/dataexplore/fonts/bootstrap/glyphicons-halflings-regular.eot new file mode 100644 index 0000000..b93a495 Binary files /dev/null and b/libcoveweb2/static/dataexplore/fonts/bootstrap/glyphicons-halflings-regular.eot differ diff --git a/libcoveweb2/static/dataexplore/fonts/bootstrap/glyphicons-halflings-regular.svg b/libcoveweb2/static/dataexplore/fonts/bootstrap/glyphicons-halflings-regular.svg new file mode 100644 index 0000000..94fb549 --- /dev/null +++ b/libcoveweb2/static/dataexplore/fonts/bootstrap/glyphicons-halflings-regular.svg @@ -0,0 +1,288 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/libcoveweb2/static/dataexplore/fonts/bootstrap/glyphicons-halflings-regular.ttf b/libcoveweb2/static/dataexplore/fonts/bootstrap/glyphicons-halflings-regular.ttf new file mode 100644 index 0000000..1413fc6 Binary files /dev/null and b/libcoveweb2/static/dataexplore/fonts/bootstrap/glyphicons-halflings-regular.ttf differ diff --git a/libcoveweb2/static/dataexplore/fonts/bootstrap/glyphicons-halflings-regular.woff b/libcoveweb2/static/dataexplore/fonts/bootstrap/glyphicons-halflings-regular.woff new file mode 100644 index 0000000..9e61285 Binary files /dev/null and b/libcoveweb2/static/dataexplore/fonts/bootstrap/glyphicons-halflings-regular.woff differ diff --git a/libcoveweb2/static/dataexplore/fonts/bootstrap/glyphicons-halflings-regular.woff2 b/libcoveweb2/static/dataexplore/fonts/bootstrap/glyphicons-halflings-regular.woff2 new file mode 100644 index 0000000..64539b5 Binary files /dev/null and b/libcoveweb2/static/dataexplore/fonts/bootstrap/glyphicons-halflings-regular.woff2 differ diff --git a/libcoveweb2/static/dataexplore/fonts/tick/tick.css b/libcoveweb2/static/dataexplore/fonts/tick/tick.css new file mode 100644 index 0000000..b39c894 --- /dev/null +++ b/libcoveweb2/static/dataexplore/fonts/tick/tick.css @@ -0,0 +1,13 @@ +/* This stylesheet generated by Transfonter (http://transfonter.org) on January 11, 2017 12:20 AM */ + +@font-face { + font-family: 'tick'; + src: url('tick.eot'); + src: url('tick.eot?#iefix') format('embedded-opentype'), + url('tick.woff2') format('woff2'), + url('tick.woff') format('woff'), + url('tick.ttf') format('truetype'), + url('tick.svg#tick') format('svg'); + font-weight: 500; + font-style: normal; +} diff --git a/libcoveweb2/static/dataexplore/fonts/tick/tick.eot b/libcoveweb2/static/dataexplore/fonts/tick/tick.eot new file mode 100644 index 0000000..ce7dfb2 Binary files /dev/null and b/libcoveweb2/static/dataexplore/fonts/tick/tick.eot differ diff --git a/libcoveweb2/static/dataexplore/fonts/tick/tick.svg b/libcoveweb2/static/dataexplore/fonts/tick/tick.svg new file mode 100644 index 0000000..5fd20d5 --- /dev/null +++ b/libcoveweb2/static/dataexplore/fonts/tick/tick.svg @@ -0,0 +1,48 @@ + + + + +Created by FontForge 20161013 at Wed Jan 11 03:13:24 2017 + By ,,, +Anonymous + + + + + + + + + + + + + + + diff --git a/libcoveweb2/static/dataexplore/fonts/tick/tick.ttf b/libcoveweb2/static/dataexplore/fonts/tick/tick.ttf new file mode 100644 index 0000000..d6bc052 Binary files /dev/null and b/libcoveweb2/static/dataexplore/fonts/tick/tick.ttf differ diff --git a/libcoveweb2/static/dataexplore/fonts/tick/tick.woff b/libcoveweb2/static/dataexplore/fonts/tick/tick.woff new file mode 100644 index 0000000..533d3db Binary files /dev/null and b/libcoveweb2/static/dataexplore/fonts/tick/tick.woff differ diff --git a/libcoveweb2/static/dataexplore/fonts/tick/tick.woff2 b/libcoveweb2/static/dataexplore/fonts/tick/tick.woff2 new file mode 100644 index 0000000..d2c61a3 Binary files /dev/null and b/libcoveweb2/static/dataexplore/fonts/tick/tick.woff2 differ diff --git a/libcoveweb2/templates/libcoveweb2/new_csvs.html b/libcoveweb2/templates/libcoveweb2/new_csvs.html new file mode 100644 index 0000000..05115cd --- /dev/null +++ b/libcoveweb2/templates/libcoveweb2/new_csvs.html @@ -0,0 +1,30 @@ +{% extends request.current_app_base_template %} +{% load i18n %} +{% load bootstrap3 %} + +{% block content %} + + +
    + +
    +
    +
    {% csrf_token %} + {% bootstrap_form forms.upload_form %} + {% buttons %} + + {% endbuttons %} + +
    +
    +
    + +{% endblock %} diff --git a/libcoveweb2/templates/libcoveweb2/new_json.html b/libcoveweb2/templates/libcoveweb2/new_json.html index 5157935..056ecf7 100644 --- a/libcoveweb2/templates/libcoveweb2/new_json.html +++ b/libcoveweb2/templates/libcoveweb2/new_json.html @@ -6,15 +6,48 @@
    -
    - {% csrf_token %} - {% bootstrap_form forms.upload_form %} - {% buttons %} - - {% endbuttons %} - + +
    +
    +
    {% csrf_token %} + {% bootstrap_form forms.upload_form %} + {% buttons %} + + {% endbuttons %} + +
    +
    +
    + + +
    + +
    +
    +
    {% csrf_token %} + {% bootstrap_form forms.text_form %} + {% buttons %} + + {% endbuttons %} + +
    +
    {% endblock %} diff --git a/libcoveweb2/templates/libcoveweb2/new_spreadsheet.html b/libcoveweb2/templates/libcoveweb2/new_spreadsheet.html index af745fd..e717d10 100644 --- a/libcoveweb2/templates/libcoveweb2/new_spreadsheet.html +++ b/libcoveweb2/templates/libcoveweb2/new_spreadsheet.html @@ -6,15 +6,27 @@
    -
    - {% csrf_token %} - {% bootstrap_form forms.upload_form %} - {% buttons %} - - {% endbuttons %} - + +
    +
    +
    {% csrf_token %} + {% bootstrap_form forms.upload_form %} + {% buttons %} + + {% endbuttons %} + +
    +
    + + {% endblock %} diff --git a/libcoveweb2/urls.py b/libcoveweb2/urls.py index d8a933a..e877bc9 100644 --- a/libcoveweb2/urls.py +++ b/libcoveweb2/urls.py @@ -6,6 +6,7 @@ urlpatterns = [ re_path(r"^new_json$", libcoveweb2.views.new_json, name="new_json"), + re_path(r"^new_csvs$", libcoveweb2.views.new_csvs, name="new_csvs"), re_path( r"^new_spreadsheet$", libcoveweb2.views.new_spreadsheet, name="new_spreadsheet" ), diff --git a/libcoveweb2/views.py b/libcoveweb2/views.py index deff4d6..d150a55 100644 --- a/libcoveweb2/views.py +++ b/libcoveweb2/views.py @@ -1,48 +1,122 @@ +from django.conf import settings from django.core.exceptions import ValidationError from django.http import HttpResponseRedirect from django.shortcuts import render from django.utils.translation import gettext_lazy as _ -from libcoveweb2.forms import NewJSONUploadForm, NewSpreadsheetUploadForm -from libcoveweb2.models import SuppliedData, SuppliedDataFile -from libcoveweb2.settings import ( - ALLOWED_JSON_CONTENT_TYPES, - ALLOWED_JSON_EXTENSIONS, - ALLOWED_SPREADSHEET_CONTENT_TYPES, - ALLOWED_SPREADSHEET_EXTENSIONS, +from libcoveweb2.forms import ( + NewCSVsUploadForm, + NewJSONTextForm, + NewJSONUploadForm, + NewSpreadsheetUploadForm, ) +from libcoveweb2.models import SuppliedData, SuppliedDataFile + +JSON_FORM_CLASSES = { + "upload_form": NewJSONUploadForm, + "text_form": NewJSONTextForm, +} def new_json(request): forms = { - "upload_form": NewJSONUploadForm(request.POST, request.FILES) + form_name: form_class() for form_name, form_class in JSON_FORM_CLASSES.items() + } + request_data = None + if request.POST: + request_data = request.POST + if request_data: + if "paste" in request_data: + form_name = "text_form" + else: + form_name = "upload_form" + forms[form_name] = JSON_FORM_CLASSES[form_name](request_data, request.FILES) + form = forms[form_name] + if form.is_valid(): + # Extra Validation + if form_name == "upload_form": + if ( + not request.FILES["file_upload"].content_type + in settings.ALLOWED_JSON_CONTENT_TYPES + ): + form.add_error( + "file_upload", "This does not appear to be a JSON file" + ) + if not [ + e + for e in settings.ALLOWED_JSON_EXTENSIONS + if str(request.FILES["file_upload"].name).lower().endswith(e) + ]: + form.add_error( + "file_upload", "This does not appear to be a JSON file" + ) + elif form_name == "text_form": + pass # TODO + + # Process + if form.is_valid(): + supplied_data = SuppliedData() + supplied_data.format = "json" + supplied_data.save() + + if form_name == "upload_form": + supplied_data.save_file(request.FILES["file_upload"]) + elif form_name == "text_form": + supplied_data.save_file_contents( + "input.json", + form.cleaned_data["paste"], + "application/json", + None, + ) + + return HttpResponseRedirect(supplied_data.get_absolute_url()) + + return render(request, "libcoveweb2/new_json.html", {"forms": forms}) + + +CSVS_FORM_CLASSES = { + "upload_form": NewCSVsUploadForm, +} + + +def new_csvs(request): + + forms = { + "upload_form": NewCSVsUploadForm(request.POST, request.FILES) if request.POST - else NewJSONUploadForm() + else NewCSVsUploadForm() } form = forms["upload_form"] if form.is_valid(): # Extra Validation - if not request.FILES["file_upload"].content_type in ALLOWED_JSON_CONTENT_TYPES: - form.add_error("file_upload", "This does not appear to be a JSON file") - if not [ - e - for e in ALLOWED_JSON_EXTENSIONS - if str(request.FILES["file_upload"].name).lower().endswith(e) - ]: - form.add_error("file_upload", "This does not appear to be a JSON file") + for field in form.file_field_names: + if request.FILES.get(field): + if ( + not request.FILES[field].content_type + in settings.ALLOWED_CSV_CONTENT_TYPES + ): + form.add_error(field, "This does not appear to be a CSV file") + if not [ + e + for e in settings.ALLOWED_CSV_EXTENSIONS + if str(request.FILES[field].name).lower().endswith(e) + ]: + form.add_error(field, "This does not appear to be a CSV file") # Process if form.is_valid(): supplied_data = SuppliedData() - supplied_data.format = "json" + supplied_data.format = "csvs" supplied_data.save() - supplied_data.save_file(request.FILES["file_upload"]) + for field in form.file_field_names: + if request.FILES.get(field): + supplied_data.save_file(request.FILES[field]) return HttpResponseRedirect(supplied_data.get_absolute_url()) - return render(request, "libcoveweb2/new_json.html", {"forms": forms}) + return render(request, "libcoveweb2/new_csvs.html", {"forms": forms}) def new_spreadsheet(request): @@ -57,12 +131,12 @@ def new_spreadsheet(request): # Extra Validation if ( not request.FILES["file_upload"].content_type - in ALLOWED_SPREADSHEET_CONTENT_TYPES + in settings.ALLOWED_SPREADSHEET_CONTENT_TYPES ): form.add_error("file_upload", "This does not appear to be a spreadsheet") if not [ e - for e in ALLOWED_SPREADSHEET_EXTENSIONS + for e in settings.ALLOWED_SPREADSHEET_EXTENSIONS if str(request.FILES["file_upload"].name).lower().endswith(e) ]: form.add_error("file_upload", "This does not appear to be a spreadsheet")