"""
 Pull building outlines (WFS), DSM, imagery, and the wide terrain DEM
 (Exports API) for the pilot from the LINZ Data Service, and save them under
 data/.

Requires a LINZ_API_KEY that may create EXPORTS, not only use the web
services (WFS/WMS/tiles) -- export job creation 401s otherwise. A key's
permissions are chosen when it is created and cannot be edited afterwards:
make a NEW key (data.linz.govt.nz -> your account -> API keys -> Create API
key) with all permissions ticked, the export/download one above all, and
put it in .env.

Usage: python src/fetch_data.py
"""

import json
import os
import shutil
import sys
import time
import zipfile
from pathlib import Path

import rasterio
import requests
from dotenv import load_dotenv
from rasterio.merge import merge

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
import config

DATA_DIR = Path(__file__).resolve().parent.parent / "data"
EXPORT_CREATE_TIMEOUT_S = 120


def fetch_building_outlines(bbox_nztm2000, api_key, layer_id=config.LINZ_BUILDING_OUTLINES_LAYER):
    """
    bbox_nztm2000: [minx, miny, maxx, maxy] in EPSG:2193 metres.

    Uses WFS 1.0.0 (not 2.0.0) -- see README "Status" for why: 2.0.0's
    typeNames/count params silently returned 0 features for every bbox
    tried here, with no error, while 1.0.0's typeName/maxFeatures works.
    """
    type_name = f"data.linz.govt.nz:layer-{layer_id}"
    bbox_str = ",".join(str(v) for v in bbox_nztm2000)
    url = "https://data.linz.govt.nz/services"
    params = {
        "service": "WFS",
        "version": "1.0.0",
        "request": "GetFeature",
        "typeName": type_name,
        "outputFormat": "json",
        "bbox": bbox_str,
        "maxFeatures": 10000,
    }
    from src.dataset_timing import transfer
    with transfer():
        resp = requests.get(f"{url};key={api_key}/wfs", params=params, timeout=60)
        resp.raise_for_status()
        data = resp.json()
    if "features" not in data:
        raise RuntimeError(f"Unexpected WFS response: {data}")
    return data


def fetch_raster(bbox_wgs84, api_key, layer_id, name, out_dir=DATA_DIR, format_key="grid"):
    """
    bbox_wgs84: [min_lon, min_lat, max_lon, max_lat]. Creates a Koordinates
    export job cropped to this extent, polls until complete, downloads and
    unzips it, then mosaics the (possibly several) 1km survey tiles into a
    single GeoTIFF at out_dir/{name}_mosaic.tif. Works for any LDS raster
    layer -- DSM/DEM are Koordinates data_type "Grid" (format_key="grid"),
    RGB aerial imagery is data_type "Raster" (format_key="raster"); the
    Exports API 400s if you send the wrong one for the layer's type.
    """
    min_lon, min_lat, max_lon, max_lat = bbox_wgs84
    body = {
        "crs": "EPSG:2193",
        "formats": {format_key: "image/tiff;subtype=geotiff"},
        "items": [{"item": f"https://data.linz.govt.nz/services/api/v1/layers/{layer_id}/"}],
        "extent": {
            "type": "Polygon",
            "coordinates": [[
                [min_lon, min_lat], [min_lon, max_lat],
                [max_lon, max_lat], [max_lon, min_lat], [min_lon, min_lat],
            ]],
        },
    }
    headers = {"Authorization": f"key {api_key}", "Content-Type": "application/json"}

    try:
        resp = requests.post("https://data.linz.govt.nz/services/api/v1/exports/",
                             headers=headers, json=body, timeout=EXPORT_CREATE_TIMEOUT_S)
    except requests.Timeout:
        raise RuntimeError(
            f"LINZ export creation timed out after {EXPORT_CREATE_TIMEOUT_S} seconds "
            f"for layer {layer_id} ({name}). Retry the build; completed rasters are "
            "reused. The request is not automatically repeated because LINZ may "
            "have created the export before the response timed out."
        ) from None
    if resp.status_code == 401:
        raise SystemExit(
            "401 from the LINZ Exports API: this LINZ_API_KEY may use the web "
            "services but not create exports. Permissions cannot be added to an "
            "existing key -- create a NEW one (data.linz.govt.nz -> your account "
            "-> API keys -> Create API key) with every permission ticked, "
            "including export/download, and put it in .env as LINZ_API_KEY."
        )
    if resp.status_code >= 400:
        # LINZ returns a JSON body naming the exact problem (bad extent, wrong
        # format for the layer type, area outside coverage) -- surface it
        # instead of a bare 400.
        #
        # SHOW THE REASON, NOT THE PREAMBLE. This truncated at 300 characters,
        # and the first 300 characters of that body are the licence and item
        # boilerplate every response carries -- so a 400 on the Wanaka DSM
        # printed a Creative Commons URL and hid why. Pull out the fields that
        # actually say what is wrong, and keep a longer tail behind them.
        detail = ""
        try:
            j = resp.json()
            bits = []
            for it in (j.get("items") or []):
                r = it.get("invalid_reasons") or it.get("reasons")
                if r:
                    bits.append(f"item: {r}")
            for k in ("error", "detail", "message", "non_field_errors",
                      "extent", "formats", "crs"):
                if j.get(k):
                    bits.append(f"{k}: {j[k]}")
            detail = " | ".join(str(b) for b in bits)
        except Exception:
            pass
        raise RuntimeError(
            f"Exports API {resp.status_code} for layer {layer_id} ({name}): "
            f"{detail or resp.text[:800]}")
    job = resp.json()
    job_url = job["url"]
    print(f"Export job {job['id']} created, polling...")

    from src.dataset_timing import phase
    with phase("export_wait"):
        while job["state"] == "processing":
            time.sleep(5)
            job = requests.get(job_url, headers=headers, timeout=30).json()
            print(f"  state={job['state']} progress={job.get('progress')}")

    if job["state"] != "complete":
        raise RuntimeError(f"Export job did not complete: {job}")

    zip_path = out_dir / f"{name}_export.zip"
    from src.dataset_timing import transfer
    with transfer():
        with requests.get(job["download_url"], headers=headers, timeout=300, stream=True) as dl:
            dl.raise_for_status()
            with open(zip_path, "wb") as f:
                for chunk in dl.iter_content(chunk_size=8 * 1024 * 1024):
                    f.write(chunk)  # streamed -- region-scale imagery zips run to GBs, far past
                    # what buffering the whole response in memory (the original approach) should carry

    with phase("local_prepare"):
        extract_dir = out_dir / name
        with zipfile.ZipFile(zip_path) as zf:
            zf.extractall(extract_dir)

        tifs = sorted(extract_dir.glob("*.tif"))
        srcs = [rasterio.open(f) for f in tifs]
        mosaic, transform = merge(srcs)
        profile = srcs[0].profile
        profile.update(height=mosaic.shape[1], width=mosaic.shape[2], transform=transform)
        mosaic_path = out_dir / f"{name}_mosaic.tif"
        temporary = mosaic_path.with_name(mosaic_path.stem + ".part.tif")
        try:
            with rasterio.open(temporary, "w", **profile) as dst:
                dst.write(mosaic)
            temporary.replace(mosaic_path)
        finally:
            for s in srcs:
                s.close()
            temporary.unlink(missing_ok=True)

    print(f"Mosaicked {len(tifs)} tiles -> {mosaic_path}")
    reclaim_export_intermediates(zip_path, extract_dir, mosaic_path)
    return mosaic_path


def reclaim_export_intermediates(zip_path, extract_dir, mosaic_path,
                                 keep=None):
    """Delete the download archive and the unpacked tiles, once the mosaic
    they produced actually exists.

    Nothing used to remove these. On 31 Aug the Queenstown rebuild was down to
    20 GB free with 11 of 25 regions still to write, while 45.5 GB of
    `*_export.zip` sat on disk across 62 files -- plus the unpacked tiles
    beside them, which are the same pixels a third time. The build would have
    died hours in, and the failure would have looked like a disk problem rather
    than a missing cleanup.

    Both inputs are pure intermediates: the zip is the raw download, the
    extracted tiles are only ever merged into the mosaic, and a re-fetch
    reproduces either. The mosaic itself is NEVER touched here -- later stages
    and the truth scorecard read it, and deleting imagery out from under a
    later stage has broken a run before.

    Deletes only when the mosaic exists and is non-empty, so a failed merge
    leaves everything in place to retry from. Set SOLAR_KEEP_EXPORTS=1 to keep
    them when debugging a bad mosaic.
    """
    if keep is None:
        keep = os.environ.get("SOLAR_KEEP_EXPORTS") == "1"
    if keep:
        print("  (SOLAR_KEEP_EXPORTS=1 -- keeping export intermediates)")
        return 0
    if not (mosaic_path.exists() and mosaic_path.stat().st_size > 0):
        print("  (mosaic missing or empty -- keeping intermediates to retry from)")
        return 0

    freed = 0
    try:
        if zip_path.exists():
            freed += zip_path.stat().st_size
            zip_path.unlink()
        if extract_dir.exists() and extract_dir.is_dir():
            freed += sum(f.stat().st_size for f in extract_dir.rglob("*")
                         if f.is_file())
            shutil.rmtree(extract_dir)
    except OSError as exc:
        # Never fail a fetch over cleanup -- a full disk is recoverable, a
        # half-fetched region is not.
        print(f"  (could not reclaim export intermediates: {exc})")
    if freed:
        print(f"  reclaimed {freed / 1024**3:.1f} GB of export intermediates")
    return freed


def main():
    load_dotenv(Path(__file__).resolve().parent.parent / ".env")
    from src.project_secrets import linz_api_key
    api_key = linz_api_key()
    if not api_key:
        raise SystemExit("LINZ_API_KEY not set -- copy .env.example to .env and fill it in")
    if config.PILOT_BBOX is None:
        raise SystemExit("config.PILOT_BBOX is not set")

    DATA_DIR.mkdir(exist_ok=True)

    print(f"Fetching building outlines for bbox {config.PILOT_BBOX_NZTM2000} (NZTM2000)...")
    buildings = fetch_building_outlines(config.PILOT_BBOX_NZTM2000, api_key)
    out_path = DATA_DIR / "building_outlines.geojson"
    out_path.write_text(json.dumps(buildings))
    print(f"Saved {len(buildings['features'])} buildings to {out_path}")

    from src.surveys import survey_for
    survey = survey_for(config.PILOT_BBOX, "pilot")
    if survey.get("elevation") == "synthetic" or not survey.get("dsm_layer"):
        raise RuntimeError("no covering LINZ DSM is available for the pilot area")
    print(f"Fetching DSM for bbox {config.PILOT_BBOX} (WGS84)...")
    fetch_raster(config.PILOT_BBOX, api_key, survey["dsm_layer"], "dsm")

    from src.fetch_dem_wide import ensure_dem_wide
    print("Ensuring wide 8m DEM for distant terrain...")
    ensure_dem_wide(api_key)

    if survey.get("imagery_layer"):
        print(f"Fetching aerial imagery for bbox {config.PILOT_BBOX} (WGS84)...")
        fetch_raster(config.PILOT_BBOX, api_key, survey["imagery_layer"],
                     "imagery", format_key="raster")
    else:
        print("No covering LINZ aerial imagery was found; continuing without imagery.")


if __name__ == "__main__":
    main()
