diff --git a/.gitignore b/.gitignore index 7f0bfa37a..510071969 100644 --- a/.gitignore +++ b/.gitignore @@ -98,4 +98,6 @@ ENV/ /site # mypy -.mypy_cache/ \ No newline at end of file +.mypy_cache/ + +cdk.out/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index edf3c63c2..7393f1f54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,7 @@ -FROM tiangolo/uvicorn-gunicorn-fastapi:python3.7 +FROM laurents/uvicorn-gunicorn-fastapi:python3.7-slim +# Ref https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker/issues/15 +# Cuts image size by 50% +# FROM tiangolo/uvicorn-gunicorn-fastapi:python3.7 ENV CURL_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt @@ -6,4 +9,4 @@ COPY README.md /app/README.md COPY titiler/ /app/titiler/ COPY setup.py /app/setup.py -RUN pip install /app/. \ No newline at end of file +RUN pip install -e /app/. --no-cache-dir diff --git a/README.md b/README.md index 1bcf21f51..4ef58641c 100644 --- a/README.md +++ b/README.md @@ -7,24 +7,120 @@ A lightweight Cloud Optimized GeoTIFF tile server. # Deployment -**To Do** +The stack is deployed by the [aws cdk](https://aws.amazon.com/cdk/) utility. It will handle tasks such as generating a docker image and packaging handlers automatically. + +1. Instal cdk and set up CDK in your AWS account - Only need once per account +```bash +$ npm install cdk -g + +$ cdk bootstrap # Deploys the CDK toolkit stack into an AWS environment +``` + +2. Install dependencies + +```bash +# Note: it's recommanded to use virtualenv +$ git clone https://github.com/developmentseed/titiler.git +$ cd titiler && pip install -e .[deploy] +``` + +3. Pre-Generate CFN template +```bash +$ cdk synth # Synthesizes and prints the CloudFormation template for this stack +``` + +4. Edit [stack/config.py](stack/config.py) + +```python +PROJECT_NAME = "titiler" +STAGE = os.environ.get("STAGE", "dev") + +# // Service config +# Min/Max Number of ECS images +MIN_ECS_INSTANCES = 2 +MAX_ECS_INSTANCES = 50 + +# CPU value | Memory value +# 256 (.25 vCPU) | 0.5 GB, 1 GB, 2 GB +# 512 (.5 vCPU) | 1 GB, 2 GB, 3 GB, 4 GB +# 1024 (1 vCPU) | 2 GB, 3 GB, 4 GB, 5 GB, 6 GB, 7 GB, 8 GB +# 2048 (2 vCPU) | Between 4 GB and 16 GB in 1-GB increments +# 4096 (4 vCPU) | Between 8 GB and 30 GB in 1-GB increments +TASK_CPU = 1024 +TASK_MEMORY = 2048 +``` + +5. Deploy +```bash +$ cdk deploy # Deploys the stack(s) named STACKS into your AWS account +``` # Test locally ```bash +$ git clone https://github.com/developmentseed/titiler.git + +$ pip install -e . $ uvicorn titiler.main:app --reload ``` - -### Docker +Or with Docker ``` $ docker-compose build $ docker-compose up ``` -## Authors -Created by [Development Seed]() +# API + +### Doc + +`:endpoint:/docs` +![](https://user-images.githubusercontent.com/10407788/78325903-011c9680-7547-11ea-853f-50e0fb0f4d92.png) + +### Tiles + +`:endpoint:/v1/{z}/{x}/{y}[@{scale}x][.{ext}]` +- **z**: Mercator tiles's zoom level. +- **x**: Mercator tiles's column. +- **y**: Mercator tiles's row. +- **scale**: Tile size scale, default is set to 1 (256x256). OPTIONAL +- **ext**: Output image format, default is set to None and will be either JPEG or PNG depending on masked value. OPTIONAL +- **url**: Cloud Optimized GeoTIFF URL. **REQUIRED** +- **bidx**: Coma (',') delimited band indexes. OPTIONAL +- **nodata**: Overwrite internal Nodata value. OPTIONAL +- **rescale**: Coma (',') delimited Min,Max bounds. OPTIONAL +- **color_formula**: rio-color formula. OPTIONAL +- **color_map**: rio-tiler color map name. OPTIONAL + +### Metadata + +`:endpoint:/v1/tilejson.json` - Get tileJSON document +- **url**: Cloud Optimized GeoTIFF URL. **REQUIRED** +- **tile_format**: Output image format, default is set to None and will be either JPEG or PNG depending on masked value. +- **tile_scale**: Tile size scale, default is set to 1 (256x256). OPTIONAL +- **kwargs**: Other options will be forwarded to the `tiles` url. + +`:endpoint:/v1/bounds` - Get general image bounds +- **url**: Cloud Optimized GeoTIFF URL. **REQUIRED** + +`:endpoint:/v1/info` - Get general image info +- **url**: Cloud Optimized GeoTIFF URL. **REQUIRED** + +`:endpoint:/v1/metadata` - Get image statistics +- **url**: Cloud Optimized GeoTIFF URL. **REQUIRED** +- **bidx**: Coma (',') delimited band indexes. OPTIONAL +- **nodata**: Overwrite internal Nodata value. OPTIONAL +- **pmin**: min percentile, default is 2. OPTIONAL +- **pmax**: max percentile, default is 98. OPTIONAL +- **max_size**: Max image size from which to calculate statistics, default is 1024. OPTIONAL +- **histogram_bins**: Histogram bins, default is 20. OPTIONAL +- **histogram_range**: Coma (',') delimited histogram bounds. OPTIONAL +## UI -## Project structure +`:endpoint:/index.html` - Full UI (histogram, predefined rescaling, ...) + +`:endpoint:/simple_viewer.html` - Simple UI (no histogram, manual rescaling, ...) + +# Project structure ``` titiler/ - titiler python module. @@ -44,9 +140,16 @@ titiler/ - titiler python module. ├── ressources/ - application ressources (enums, constants, ...). ├── templates/ - html/xml models. ├── main.py - FastAPI application creation and configuration. - └── utils.py - utility functions. -``` + ├── utils.py - utility functions. + │ +stack/ + ├── app.py - AWS Stack definition (vpc, cluster, ecs, alb ...) + ├── config.py - Optional parameters for the stack definition [EDIT THIS] + │ +OpenAPI/ + └── openapi.json - OpenAPI document. +``` ## Contribution & Development @@ -67,3 +170,7 @@ This repo is set to use `pre-commit` to run *my-py*, *flake8*, *pydocstring* and ```bash $ pre-commit install ``` + +## Authors +Created by [Development Seed]() + diff --git a/cdk.json b/cdk.json new file mode 100644 index 000000000..12a60e800 --- /dev/null +++ b/cdk.json @@ -0,0 +1,3 @@ +{ + "app": "python3 stack/app.py" +} \ No newline at end of file diff --git a/setup.py b/setup.py index 51a74f335..72b0b8a1d 100644 --- a/setup.py +++ b/setup.py @@ -16,9 +16,17 @@ extra_reqs = { "dev": ["pytest", "pytest-cov", "pytest-asyncio", "pre-commit"], "server": ["uvicorn", "click==7.0"], + "deploy": [ + "aws-cdk.core", + "aws-cdk.aws_ecs", + "aws-cdk.aws_ec2", + "aws-cdk.aws_autoscaling", + "aws-cdk.aws_ecs_patterns", + ], "test": ["mock", "pytest", "pytest-cov", "pytest-asyncio", "requests"], } + setup( name="titiler", version="0.1.0", diff --git a/stack/__init__.py b/stack/__init__.py new file mode 100644 index 000000000..4955682f0 --- /dev/null +++ b/stack/__init__.py @@ -0,0 +1 @@ +"""AWS App.""" diff --git a/stack/app.py b/stack/app.py new file mode 100644 index 000000000..a49c5d459 --- /dev/null +++ b/stack/app.py @@ -0,0 +1,118 @@ +"""Construct App.""" + +from typing import Any, Union + +import os + +from aws_cdk import ( + core, + aws_ec2 as ec2, + aws_ecs as ecs, + aws_ecs_patterns as ecs_patterns, +) + +import config + + +class titilerStack(core.Stack): + """Titiler ECS Fargate Stack.""" + + def __init__( + self, + scope: core.Construct, + id: str, + cpu: Union[int, float] = 256, + memory: Union[int, float] = 512, + mincount: int = 1, + maxcount: int = 50, + code_dir: str = "./", + **kwargs: Any, + ) -> None: + """Define stack.""" + super().__init__(scope, id, *kwargs) + + vpc = ec2.Vpc(self, f"{id}-vpc", max_azs=2) + + cluster = ecs.Cluster(self, f"{id}-cluster", vpc=vpc) + + fargate_service = ecs_patterns.ApplicationLoadBalancedFargateService( + self, + f"{id}-service", + cluster=cluster, + cpu=cpu, + memory_limit_mib=memory, + desired_count=mincount, + public_load_balancer=True, + listener_port=80, + task_image_options=dict( + image=ecs.ContainerImage.from_asset( + code_dir, exclude=["cdk.out", ".git"] + ), + container_port=80, + environment=dict( + CPL_TMPDIR="/tmp", + GDAL_CACHEMAX="25%", + GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR", + GDAL_HTTP_MERGE_CONSECUTIVE_RANGES="YES", + GDAL_HTTP_MULTIPLEX="YES", + GDAL_HTTP_VERSION="2", + MODULE_NAME="titiler.main", + PYTHONWARNINGS="ignore", + VARIABLE_NAME="app", + VSI_CACHE="TRUE", + VSI_CACHE_SIZE="1000000", + WORKERS_PER_CORE="5", + LOG_LEVEL="error", + ), + ), + ) + + scalable_target = fargate_service.service.auto_scale_task_count( + min_capacity=mincount, max_capacity=maxcount + ) + + # https://github.com/awslabs/aws-rails-provisioner/blob/263782a4250ca1820082bfb059b163a0f2130d02/lib/aws-rails-provisioner/scaling.rb#L343-L387 + scalable_target.scale_on_request_count( + "RequestScaling", + requests_per_target=50, + scale_in_cooldown=core.Duration.seconds(240), + scale_out_cooldown=core.Duration.seconds(30), + target_group=fargate_service.target_group, + ) + + # scalable_target.scale_on_cpu_utilization( + # "CpuScaling", target_utilization_percent=70, + # ) + + fargate_service.service.connections.allow_from_any_ipv4( + port_range=ec2.Port( + protocol=ec2.Protocol.ALL, + string_representation="All port 80", + from_port=80, + ), + description="Allows traffic on port 80 from NLB", + ) + + +app = core.App() + +# Tag infrastructure +for key, value in { + "Project": config.PROJECT_NAME, + "Stack": config.STAGE, + "Owner": os.environ.get("OWNER"), + "Client": os.environ.get("CLIENT"), +}.items(): + if value: + core.Tag.add(app, key, value) + +stackname = f"{config.PROJECT_NAME}-{config.STAGE}" +titilerStack( + app, + stackname, + cpu=config.TASK_CPU, + memory=config.TASK_MEMORY, + mincount=config.MIN_ECS_INSTANCES, + maxcount=config.MAX_ECS_INSTANCES, +) +app.synth() diff --git a/stack/config.py b/stack/config.py new file mode 100644 index 000000000..acf851b9e --- /dev/null +++ b/stack/config.py @@ -0,0 +1,20 @@ +"""STACK Configs.""" + +import os + +PROJECT_NAME = "titiler" +STAGE = os.environ.get("STAGE", "dev") + +# // Service config +# Min/Max Number of ECS images +MIN_ECS_INSTANCES = 2 +MAX_ECS_INSTANCES = 50 + +# CPU value | Memory value +# 256 (.25 vCPU) | 0.5 GB, 1 GB, 2 GB +# 512 (.5 vCPU) | 1 GB, 2 GB, 3 GB, 4 GB +# 1024 (1 vCPU) | 2 GB, 3 GB, 4 GB, 5 GB, 6 GB, 7 GB, 8 GB +# 2048 (2 vCPU) | Between 4 GB and 16 GB in 1-GB increments +# 4096 (4 vCPU) | Between 8 GB and 30 GB in 1-GB increments +TASK_CPU = 1024 +TASK_MEMORY = 2048 diff --git a/titiler/api/__init__.py b/titiler/api/__init__.py index a9f37fa85..834e402ac 100644 --- a/titiler/api/__init__.py +++ b/titiler/api/__init__.py @@ -1 +1 @@ -"""titiler.""" +"""titiler.api""" diff --git a/titiler/api/api_v1/__init__.py b/titiler/api/api_v1/__init__.py index a9f37fa85..18d93add2 100644 --- a/titiler/api/api_v1/__init__.py +++ b/titiler/api/api_v1/__init__.py @@ -1 +1 @@ -"""titiler.""" +"""titiler.api.api_v1""" diff --git a/titiler/api/api_v1/endpoints/__init__.py b/titiler/api/api_v1/endpoints/__init__.py index a9f37fa85..a142702d9 100644 --- a/titiler/api/api_v1/endpoints/__init__.py +++ b/titiler/api/api_v1/endpoints/__init__.py @@ -1 +1 @@ -"""titiler.""" +"""titiler.api.api_v1.endpoints""" diff --git a/titiler/api/api_v1/endpoints/metadata.py b/titiler/api/api_v1/endpoints/metadata.py index e73d809dd..a35a51b8d 100644 --- a/titiler/api/api_v1/endpoints/metadata.py +++ b/titiler/api/api_v1/endpoints/metadata.py @@ -19,8 +19,9 @@ from titiler.core import config from titiler.models.mapbox import TileJSON from titiler.ressources.enums import ImageType +from titiler.api.utils import info as cogInfo - +_info = partial(run_in_threadpool, cogInfo) _bounds = partial(run_in_threadpool, cogeo.bounds) _metadata = partial(run_in_threadpool, cogeo.metadata) _spatial_info = partial(run_in_threadpool, cogeo.spatial_info) @@ -96,6 +97,16 @@ async def bounds( return await _bounds(url) +@router.get("/info", responses={200: {"description": "Return basic info on COG."}}) +async def info( + response: Response, + url: str = Query(..., description="Cloud Optimized GeoTIFF URL."), +): + """Handle /info requests.""" + response.headers["Cache-Control"] = "max-age=3600" + return await _info(url) + + @router.get( "/metadata", responses={200: {"description": "Return the metadata of the COG."}} ) diff --git a/titiler/api/utils.py b/titiler/api/utils.py index eeaa21414..5680bd1f7 100644 --- a/titiler/api/utils.py +++ b/titiler/api/utils.py @@ -1,6 +1,6 @@ """titiler.api.utils.""" -from typing import Any, Optional +from typing import Any, Dict, Optional import json import hashlib @@ -9,6 +9,12 @@ from starlette.requests import Request +# Temporary +import rasterio +from rasterio.warp import transform_bounds +from rio_tiler import constants +from rio_tiler.utils import has_alpha_band, has_mask_band +from rio_tiler.mercator import get_zooms from rio_color.operations import parse_operations from rio_color.utils import scale_dtype, to_math_type @@ -58,3 +64,70 @@ def postprocess( tile = scale_dtype(ops(to_math_type(tile)), numpy.uint8) return tile + + +# from rio-tiler 2.0a5 +def info(address: str) -> Dict: + """ + Return simple metadata about the file. + + Attributes + ---------- + address : str or PathLike object + A dataset path or URL. Will be opened in "r" mode. + + Returns + ------- + out : dict. + + """ + with rasterio.open(address) as src_dst: + minzoom, maxzoom = get_zooms(src_dst) + bounds = transform_bounds( + src_dst.crs, constants.WGS84_CRS, *src_dst.bounds, densify_pts=21 + ) + center = [(bounds[0] + bounds[2]) / 2, (bounds[1] + bounds[3]) / 2, minzoom] + + def _get_descr(ix): + """Return band description.""" + name = src_dst.descriptions[ix - 1] + if not name: + name = "band{}".format(ix) + return name + + band_descriptions = [(ix, _get_descr(ix)) for ix in src_dst.indexes] + tags = [(ix, src_dst.tags(ix)) for ix in src_dst.indexes] + + other_meta = dict() + if src_dst.scales[0] and src_dst.offsets[0]: + other_meta.update(dict(scale=src_dst.scales[0])) + other_meta.update(dict(offset=src_dst.offsets[0])) + + if has_alpha_band(src_dst): + nodata_type = "Alpha" + elif has_mask_band(src_dst): + nodata_type = "Mask" + elif src_dst.nodata is not None: + nodata_type = "Nodata" + else: + nodata_type = "None" + + try: + cmap = src_dst.colormap(1) + other_meta.update(dict(colormap=cmap)) + except ValueError: + pass + + return dict( + address=address, + bounds=bounds, + center=center, + minzoom=minzoom, + maxzoom=maxzoom, + band_metadata=tags, + band_descriptions=band_descriptions, + dtype=src_dst.meta["dtype"], + colorinterp=[src_dst.colorinterp[ix - 1].name for ix in src_dst.indexes], + nodata_type=nodata_type, + **other_meta, + ) diff --git a/titiler/core/__init__.py b/titiler/core/__init__.py index a9f37fa85..454a83dd6 100644 --- a/titiler/core/__init__.py +++ b/titiler/core/__init__.py @@ -1 +1 @@ -"""titiler.""" +"""titiler.core""" diff --git a/titiler/core/config.py b/titiler/core/config.py index 221737cbe..748a82005 100644 --- a/titiler/core/config.py +++ b/titiler/core/config.py @@ -15,6 +15,6 @@ DISABLE_CACHE = os.getenv("DISABLE_CACHE") MEMCACHE_HOST = os.environ.get("MEMCACHE_HOST") -MEMCACHE_PORT = os.environ.get("MEMCACHE_PORT", 11211) +MEMCACHE_PORT = int(os.environ.get("MEMCACHE_PORT", 11211)) MEMCACHE_USERNAME = os.environ.get("MEMCACHE_USERNAME") MEMCACHE_PASSWORD = os.environ.get("MEMCACHE_PASSWORD") diff --git a/titiler/db/__init__.py b/titiler/db/__init__.py new file mode 100644 index 000000000..a9072eaba --- /dev/null +++ b/titiler/db/__init__.py @@ -0,0 +1 @@ +"""titiler.db""" diff --git a/titiler/db/memcache.py b/titiler/db/memcache.py index a33176265..55317d368 100644 --- a/titiler/db/memcache.py +++ b/titiler/db/memcache.py @@ -4,6 +4,8 @@ from bmemcached import Client +from titiler.ressources.enums import ImageType + class CacheLayer(object): """Memcache Wrapper.""" @@ -18,7 +20,7 @@ def __init__( """Init Cache Layer.""" self.client = Client((f"{host}:{port}",), user, password) - def get_image_from_cache(self, img_hash: str) -> Tuple[bytes, str]: + def get_image_from_cache(self, img_hash: str) -> Tuple[bytes, ImageType]: """ Get image body from cache layer. @@ -39,7 +41,7 @@ def get_image_from_cache(self, img_hash: str) -> Tuple[bytes, str]: return content, ext def set_image_cache( - self, img_hash: str, body: Tuple[bytes, str], timeout: int = 432000 + self, img_hash: str, body: Tuple[bytes, ImageType], timeout: int = 432000 ) -> bool: """ Set base64 encoded image body in cache layer. diff --git a/titiler/main.py b/titiler/main.py index 2e3610101..6966732c4 100644 --- a/titiler/main.py +++ b/titiler/main.py @@ -1,4 +1,5 @@ """titiler app.""" +from typing import Any, Dict from fastapi import FastAPI from starlette.requests import Request @@ -16,7 +17,7 @@ templates = Jinja2Templates(directory="titiler/templates") if config.MEMCACHE_HOST and not config.DISABLE_CACHE: - kwargs = { + kwargs: Dict[str, Any] = { k: v for k, v in zip( ["port", "user", "password"], @@ -83,6 +84,26 @@ def index(request: Request): ) +@app.get( + "/simple_viewer.html", + responses={200: {"content": {"application/hmtl": {}}}}, + response_class=HTMLResponse, +) +def simple(request: Request): + """Demo Page.""" + scheme = request.url.scheme + host = request.headers["host"] + if config.API_VERSION_STR: + host += config.API_VERSION_STR + endpoint = f"{scheme}://{host}" + + return templates.TemplateResponse( + "simple.html", + {"request": request, "endpoint": endpoint}, + media_type="text/html", + ) + + @app.get("/ping", description="Health Check") def ping(): """Health check.""" diff --git a/titiler/models/__init__.py b/titiler/models/__init__.py index a9f37fa85..a6888bb16 100644 --- a/titiler/models/__init__.py +++ b/titiler/models/__init__.py @@ -1 +1 @@ -"""titiler.""" +"""titiler.models""" diff --git a/titiler/ressources/__init__.py b/titiler/ressources/__init__.py index b74a1e0cc..5d91b8219 100644 --- a/titiler/ressources/__init__.py +++ b/titiler/ressources/__init__.py @@ -1 +1 @@ -"""titiler ressources.""" +"""titiler.ressources.""" diff --git a/titiler/templates/__init__.py b/titiler/templates/__init__.py new file mode 100644 index 000000000..47bf20895 --- /dev/null +++ b/titiler/templates/__init__.py @@ -0,0 +1 @@ +"""titiler.templates.""" diff --git a/titiler/templates/index.html b/titiler/templates/index.html index c1d5858bc..a5931f0a0 100644 --- a/titiler/templates/index.html +++ b/titiler/templates/index.html @@ -210,6 +210,14 @@ +
+
+
Enter COG url
+ + +
+
+