Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions processing/app/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,12 @@
# Timeout (seconds) for downloading a remote source file.
DOWNLOAD_TIMEOUT = float(os.environ.get("LITE_DOWNLOAD_TIMEOUT", "60"))

# How many times to try downloading a source from scratch - the first,
# plus this many retries - if the connection drops before it's done (e.g.
# "peer closed connection"), and how long to wait between tries.
DOWNLOAD_RETRIES = int(os.environ.get("LITE_DOWNLOAD_RETRIES", "3"))
DOWNLOAD_RETRY_DELAY = float(os.environ.get("LITE_DOWNLOAD_RETRY_DELAY", "2"))

# S3 (or S3-compatible, e.g. MinIO) credentials used to presign GET URLs
# for `s3://bucket/key` sources. Left unset, s3:// sources are rejected.
S3_ENDPOINT_URL = os.environ.get("S3_ENDPOINT_URL") or None
Expand Down Expand Up @@ -35,5 +41,13 @@
# (a mosaic's outputs). Generous: a merged mosaic can be gigabytes.
UPLOAD_TIMEOUT = float(os.environ.get("LITE_UPLOAD_TIMEOUT", "1800"))

# How many times to try uploading a result file from scratch - the first,
# plus this many retries - if the connection drops, or the bucket answers
# with a server error (e.g. a transient 504), before it's done, and how
# long to wait between tries. Not retried for a client error (4xx, e.g. an
# expired presigned URL): trying again won't fix that.
UPLOAD_RETRIES = int(os.environ.get("LITE_UPLOAD_RETRIES", "3"))
UPLOAD_RETRY_DELAY = float(os.environ.get("LITE_UPLOAD_RETRY_DELAY", "2"))

# How many of a mosaic's tiles convert at once, within its one job.
MOSAIC_TILE_WORKERS = int(os.environ.get("LITE_MOSAIC_TILE_WORKERS", "2"))
12 changes: 7 additions & 5 deletions processing/app/services/tiff_mosaic.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,22 +177,24 @@ def _mosaic(tiles, vrt, merged, thumbnail_output, job_id, workdir) -> dict:
"id": tile["id"],
"bbox": tile["bbox"],
"data": upload_file(
tile["data"], spec["data_upload"], COG_MEDIA_TYPE
tile["data"], spec["data_upload"], COG_MEDIA_TYPE, job_id
),
}
if not merged:
entry["web"] = upload_file(
tile["web"], spec["web_upload"], COG_MEDIA_TYPE
tile["web"], spec["web_upload"], COG_MEDIA_TYPE, job_id
)
result["tiles"].append(entry)
result["vrt"] = upload_file(vrt_path, vrt["upload"], VRT_MEDIA_TYPE)
result["vrt"] = upload_file(
vrt_path, vrt["upload"], VRT_MEDIA_TYPE, job_id)
if merged:
result["merged"] = upload_file(
merged_path, merged["upload"], COG_MEDIA_TYPE
merged_path, merged["upload"], COG_MEDIA_TYPE, job_id
)
if has_thumbnail:
result["thumbnail"] = upload_file(
thumbnail_path, thumbnail_output["upload"], thumbnail.MEDIA_TYPE
thumbnail_path, thumbnail_output["upload"],
thumbnail.MEDIA_TYPE, job_id
)
_progress(job_id, f"Mosaic of {total} tiles uploaded", 1.0)
return result
67 changes: 38 additions & 29 deletions processing/app/utils/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@

import os
import shutil
import time

import httpx

from app.config import DOWNLOAD_TIMEOUT
from app.config import DOWNLOAD_RETRIES, DOWNLOAD_RETRY_DELAY, DOWNLOAD_TIMEOUT
from app.errors import ConversionError

GB = 1024**3
Expand Down Expand Up @@ -50,36 +51,44 @@ def download_url(url: str, dest_path: str) -> None:
There's no fixed size limit: the disk is. A source that says how big
it is (Content-Length) is refused up front if it won't fit; any source
is stopped if, while it streams in, free space falls to the reserve
(see disk_reserve). A partial download is removed.
(see disk_reserve). A partial download is removed. A connection that
drops before it's done (refused, reset, or otherwise never finishing -
not a real HTTP error response) is retried from scratch up to
DOWNLOAD_RETRIES times, DOWNLOAD_RETRY_DELAY apart: the URL is a plain
GET, safe to redo whole.
"""
directory = os.path.dirname(dest_path) or "."
try:
with httpx.stream(
"GET", url, timeout=DOWNLOAD_TIMEOUT, follow_redirects=True
) as response:
if response.status_code >= 400:
raise ConversionError(
400,
f"Failed to download source: "
f"HTTP {response.status_code}",
)
declared = response.headers.get("Content-Length")
if declared and declared.isdigit():
_check_space(directory, int(declared))
unchecked = 0
with open(dest_path, "wb") as f:
for chunk in response.iter_bytes():
f.write(chunk)
unchecked += len(chunk)
if unchecked >= CHECK_EVERY:
unchecked = 0
_check_space(directory)
except httpx.HTTPError as e:
_remove(dest_path)
raise ConversionError(400, f"Failed to download source: {e}")
except ConversionError:
_remove(dest_path)
raise
for attempt in range(1, DOWNLOAD_RETRIES + 1):
try:
with httpx.stream(
"GET", url, timeout=DOWNLOAD_TIMEOUT, follow_redirects=True
) as response:
if response.status_code >= 400:
raise ConversionError(
400,
f"Failed to download source: "
f"HTTP {response.status_code}",
)
declared = response.headers.get("Content-Length")
if declared and declared.isdigit():
_check_space(directory, int(declared))
unchecked = 0
with open(dest_path, "wb") as f:
for chunk in response.iter_bytes():
f.write(chunk)
unchecked += len(chunk)
if unchecked >= CHECK_EVERY:
unchecked = 0
_check_space(directory)
return
except httpx.HTTPError as e:
_remove(dest_path)
if attempt == DOWNLOAD_RETRIES:
raise ConversionError(400, f"Failed to download source: {e}")
time.sleep(DOWNLOAD_RETRY_DELAY)
except ConversionError:
_remove(dest_path)
raise


def _remove(path: str) -> None:
Expand Down
66 changes: 46 additions & 20 deletions processing/app/utils/upload.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@

import hashlib
import os
import time
from typing import List, Optional, Tuple

import httpx

from app import jobs
from app.config import UPLOAD_TIMEOUT
from app.config import UPLOAD_RETRIES, UPLOAD_RETRY_DELAY, UPLOAD_TIMEOUT
from app.errors import ConversionError

CHUNK_SIZE = 1024 * 1024
Expand All @@ -20,32 +21,56 @@ def _hashed_chunks(path: str, digest):
yield chunk


def upload_file(path: str, url: str, media_type: str) -> dict:
def upload_file(
path: str, url: str, media_type: str, job_id: Optional[str] = None
) -> dict:
"""PUT path to url, returning {'size', 'sha256'} of the bytes sent.

The caller presigns url for the object's final key, so this service
writes straight to the caller's bucket without holding its
credentials (as with presigned GET sources). The SHA-256 is computed
from the very bytes uploaded, for the caller's checksums.

A connection that drops, or a server error (5xx - e.g. a transient
504), is retried from scratch up to UPLOAD_RETRIES times,
UPLOAD_RETRY_DELAY apart - the PUT is safe to redo whole, and
`job_id` (if given) is told so the caller sees why it's taking a
moment longer, rather than it looking stuck. A client error (4xx) is
raised at once: retrying an expired presigned URL won't fix it.
"""
size = os.path.getsize(path)
digest = hashlib.sha256()
try:
response = httpx.put(
url,
content=_hashed_chunks(path, digest),
headers={"Content-Type": media_type, "Content-Length": str(size)},
timeout=UPLOAD_TIMEOUT,
)
except httpx.HTTPError as e:
raise ConversionError(502, f"Failed to upload result: {e}")
if response.status_code >= 400:
raise ConversionError(
502,
f"Failed to upload result: HTTP {response.status_code} "
f"{response.text[:200]}",
)
return {"size": size, "sha256": digest.hexdigest()}
last_error = None
for attempt in range(1, UPLOAD_RETRIES + 1):
digest = hashlib.sha256()
try:
response = httpx.put(
url,
content=_hashed_chunks(path, digest),
headers={
"Content-Type": media_type, "Content-Length": str(size)
},
timeout=UPLOAD_TIMEOUT,
)
except httpx.HTTPError as e:
last_error = f"Failed to upload result: {e}"
else:
if response.status_code < 400:
return {"size": size, "sha256": digest.hexdigest()}
last_error = (
f"Failed to upload result: HTTP {response.status_code} "
f"{response.text[:200]}"
)
if response.status_code < 500:
raise ConversionError(502, last_error)
if attempt == UPLOAD_RETRIES:
raise ConversionError(502, last_error)
if job_id:
jobs.update_detail(
job_id,
f"{last_error} - will retry ({attempt}/{UPLOAD_RETRIES})",
progress=1.0,
)
time.sleep(UPLOAD_RETRY_DELAY)


def upload_outputs(
Expand Down Expand Up @@ -79,7 +104,8 @@ def upload_outputs(
progress=1.0,
)
target = targets[(file["layer"], file["role"])]
sent = upload_file(file["path"], target["url"], target["content_type"])
sent = upload_file(
file["path"], target["url"], target["content_type"], job_id)
layers.setdefault(file["layer"], {})[file["role"]] = {
**sent,
"info": file.get("info") or {},
Expand Down
Loading