246 lines
9.6 KiB
Python
246 lines
9.6 KiB
Python
import logging
|
|
import requests
|
|
import requests.adapters
|
|
import json
|
|
import typing
|
|
import urllib.parse
|
|
|
|
from django.conf import settings
|
|
from django.contrib.auth.decorators import login_required
|
|
from django.shortcuts import render, get_object_or_404
|
|
from django.http import JsonResponse, HttpResponseRedirect
|
|
from django.urls import reverse
|
|
from django.views.decorators.http import require_http_methods
|
|
|
|
from mydata_benchmarks.models import Benchmark
|
|
from mydata_benchmarks.decorators import client_token_required
|
|
from .. import opendata, versions
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
@require_http_methods(["POST"])
|
|
@client_token_required
|
|
def benchmark_submit_view(request) -> JsonResponse:
|
|
my_log = log.getChild('benchmark_submit_view')
|
|
|
|
# TODO change the order of submission. First we should store the data on mydata, then
|
|
# we index it (asyncronously?).
|
|
|
|
headers = {"Authorization": "Token " + settings.BLENDER_OPENDATA['TOKEN']}
|
|
|
|
# Forward the remote address to Open Data, so that Sentry logs etc. include the
|
|
# original client's IP address.
|
|
remote_addr = request.META.get('REMOTE_ADDR')
|
|
if remote_addr:
|
|
headers["X-Forwarded-For"] = remote_addr
|
|
|
|
try:
|
|
benchmark_data = json.loads(request.body)
|
|
except ValueError as ex:
|
|
my_log.info('User %s submitted invalid data: %s', request.user, ex)
|
|
# TODO(Sybren): give feedback with validation errors to the caller.
|
|
return JsonResponse({'message': 'The submitted data is not valid.'}, status=422)
|
|
|
|
# Check this before forwarding the request to Open Data, so that an
|
|
# out-of-date warning in the log is logged before other potential errors.
|
|
client_version, client_outdated = _is_client_version_outdated(request, benchmark_data)
|
|
|
|
url = opendata.benchmarks_api_url()
|
|
sess = opendata.session()
|
|
my_log.debug('Sending benchmark to %s', url)
|
|
try:
|
|
r = sess.post(url, json=benchmark_data, verify=True, headers=headers)
|
|
except requests.exceptions.ConnectionError as ex:
|
|
# Just continue with the submission, but without benchmark ID.
|
|
# This will serve as an indication to the cronjob to retry submission to Open Data later.
|
|
my_log.warning('Unable to reach %s: %s', url, ex)
|
|
benchmark_id = ''
|
|
manage_id = ''
|
|
else:
|
|
if r.status_code in {401, 403}:
|
|
my_log.error('Our token was rejected by OpenData, configure BLENDER_OPENDATA["token"]:'
|
|
' %s', r.content)
|
|
return JsonResponse({'message': 'Server configuration error, try again later.'},
|
|
status=503)
|
|
|
|
if r.status_code != 201:
|
|
my_log.info('Response code %d received from %s: %s', r.status_code, url, r.text)
|
|
error_resp = {'message': 'The submitted data is not valid'}
|
|
if r.headers.get('Content-Type') == 'application/json':
|
|
error_resp.update(r.json())
|
|
return JsonResponse(error_resp, status=r.status_code)
|
|
|
|
response = r.json()
|
|
benchmark_id = response['benchmark_id']
|
|
manage_id = response['manage_id']
|
|
my_log.debug('Indexed benchmark %r at Open Data', benchmark_id)
|
|
|
|
# Create local Benchmark sample
|
|
benchmark = Benchmark.objects.create(
|
|
user=request.user,
|
|
benchmark_id=benchmark_id,
|
|
manage_id=manage_id,
|
|
data_raw=benchmark_data,
|
|
)
|
|
|
|
my_log.info('Stored benchmark %r = %r', benchmark.pk, benchmark_id)
|
|
|
|
# Construct the response
|
|
payload = {'benchmark_id': benchmark_id}
|
|
benchmark_url = benchmark.get_absolute_url()
|
|
if benchmark_id:
|
|
payload['message'] = 'All was well.'
|
|
location = benchmark_url
|
|
else:
|
|
payload['message'] = 'Your submission was queued and may take some time to show up.'
|
|
location = reverse('submission_queued', kwargs={'pk': benchmark.pk})
|
|
|
|
if client_outdated:
|
|
payload['message'] = 'Your Benchmark Client is outdated, please download the latest version'
|
|
outdated_url = reverse('client_outdated', kwargs={
|
|
'current_version': client_version or 'unknown'
|
|
})
|
|
location = f'{outdated_url}?next={urllib.parse.quote(location)}'
|
|
|
|
response = JsonResponse(payload, status=201)
|
|
response['Location'] = request.build_absolute_uri(location)
|
|
return response
|
|
|
|
|
|
def _is_client_version_outdated(request, benchmark_data: dict) -> typing.Tuple[str, bool]:
|
|
"""Check the version of the Benchmark Client.
|
|
|
|
:return: the client version and whether it's outdated.
|
|
"""
|
|
|
|
try:
|
|
client_version = benchmark_data['data']['benchmark_client']['client_version']
|
|
except KeyError:
|
|
client_version = ''
|
|
|
|
client_is_outdated = versions.is_benchmark_client_outdated(client_version)
|
|
if client_is_outdated:
|
|
log.info('User %s submitted using an outdated client version %r', request.user,
|
|
client_version)
|
|
|
|
return client_version, client_is_outdated
|
|
|
|
|
|
@require_http_methods(["POST"])
|
|
@login_required
|
|
def benchmark_delete_bulk_view(request):
|
|
"""Get a list of sample ids and remove them.
|
|
|
|
First the samples will be removed from the Open Data platform, and on
|
|
success they will also be removed from My Data.
|
|
"""
|
|
# Lists from jQuery are provided with a [] suffix)
|
|
samples_str = request.POST.getlist('samples[]')
|
|
|
|
url_base = opendata.benchmarks_api_url()
|
|
|
|
if not samples_str:
|
|
return JsonResponse({'message': 'No samples provided for deletion'}, status=400)
|
|
|
|
samples_deleted = []
|
|
samples_not_deleted = []
|
|
|
|
# Build a list with only the valid samples
|
|
samples = []
|
|
for sample_str in samples_str:
|
|
try:
|
|
int_sample_id = int(sample_str)
|
|
except ValueError:
|
|
log.error('Trying to delete invalid sample: %r', sample_str)
|
|
samples_not_deleted.append({'key': sample_str, 'message': 'Not valid',
|
|
'status': 400})
|
|
else:
|
|
samples.append(int_sample_id)
|
|
|
|
# Create a session with a retry policy to handle the case when OpenData is temporarily not
|
|
# available
|
|
requests_session = requests.Session()
|
|
requests_session.mount('https://', requests.adapters.HTTPAdapter(max_retries=5))
|
|
requests_session.mount('http://', requests.adapters.HTTPAdapter(max_retries=5))
|
|
|
|
headers = {'Authorization': 'Bearer ' + settings.BLENDER_OPENDATA['TOKEN']}
|
|
|
|
for sample_id in samples:
|
|
try:
|
|
benchmark = request.user.benchmarks.get(pk=sample_id)
|
|
except Benchmark.DoesNotExist:
|
|
log.error('Sample id %i was not found while trying to delete it')
|
|
samples_not_deleted.append({'key': sample_id,
|
|
'message': 'Not found', 'status': 400})
|
|
continue
|
|
|
|
if benchmark.benchmark_id and benchmark.manage_id:
|
|
url_benchmark = urllib.parse.urljoin(url_base, str(benchmark.manage_id))
|
|
try:
|
|
resp = requests_session.delete(url_benchmark, headers=headers)
|
|
except requests.exceptions.ConnectionError as ex:
|
|
log.error('Connection error to Open Data trying to delete sample %r: %s',
|
|
benchmark.benchmark_id, ex)
|
|
samples_not_deleted.append({
|
|
'key': sample_id,
|
|
'message': f'Could not delete benchmark {benchmark.id} from Open Data '
|
|
f'due to a connection error. Please retry in a few minutes.',
|
|
'status': 500})
|
|
break
|
|
|
|
if resp.status_code != 204:
|
|
log.error('Error deleting sample %r at %s: %s',
|
|
benchmark.benchmark_id, url_base, resp.json())
|
|
samples_not_deleted.append({'key': sample_id,
|
|
'message': f'Server error on {url_base} with code '
|
|
f'{resp.status_code}',
|
|
'status': 500})
|
|
continue
|
|
else:
|
|
log.debug('Sample id %r was not yet synced to Open Data while trying to delete it',
|
|
sample_id)
|
|
|
|
benchmark.delete()
|
|
log.debug('Deleted sample %i' % sample_id)
|
|
samples_deleted.append(sample_id)
|
|
|
|
results = {
|
|
'samplesDeleted': samples_deleted,
|
|
'samplesNotDeleted': samples_not_deleted,
|
|
}
|
|
|
|
# Display message about successful deletion of samples
|
|
# Notifications about failed deletions are done with toaster on the client side
|
|
samples_deleted_count = len(results['samplesDeleted'])
|
|
if samples_deleted_count == 0:
|
|
message = None
|
|
elif samples_deleted_count == 1:
|
|
message = 'Sample deleted'
|
|
else:
|
|
message = f'{samples_deleted_count} samples deleted'
|
|
|
|
return JsonResponse({'message': message, 'results': results}, status=200)
|
|
|
|
|
|
def outdated_view(request, current_version: str):
|
|
"""Call To Action to update client; allow redirect to ?next=xxx param."""
|
|
|
|
next_url: str = request.GET.get('next', '')
|
|
return render(request, 'benchmark_client_outdated.html',
|
|
{'next': next_url,
|
|
'current_version': current_version})
|
|
|
|
|
|
@login_required
|
|
def submission_queued(request, pk: int):
|
|
"""Notify user that the benchmark submission has been queued."""
|
|
|
|
benchmark = get_object_or_404(request.user.benchmarks, pk=pk)
|
|
if benchmark.benchmark_id:
|
|
# The benchmark has been submitted, so just redirect to the benchmark on Open Data.
|
|
benchmark_url = benchmark.get_absolute_url()
|
|
return HttpResponseRedirect(benchmark_url)
|
|
|
|
return render(request, 'submission_queued.html', {'benchmark': benchmark})
|