From 309ae98ac33f18a25fb47f14124fa65a5db629f0 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Mon, 29 May 2023 19:26:26 +0200 Subject: [PATCH 01/10] Use asymmetric jwt key instead of doing user management --- dummy_token_generator.py | 77 ++++++ pyproject.toml | 2 +- src/bartender/__main__.py | 36 --- src/bartender/db/dao/job_dao.py | 7 +- src/bartender/db/dao/user_dao.py | 84 ------- .../versions/2022-08-30-15-39_300833a7a4c8.py | 16 +- .../versions/2022-08-30-17-20_ef872771a841.py | 31 +-- .../versions/2022-09-09-14-54_a31176b2cffa.py | 5 +- .../versions/2023-03-02-08-12_3226c12cc7f8.py | 11 +- .../versions/2023-04-03-16-51_cf2424f395bc.py | 12 - src/bartender/db/models/job_model.py | 11 +- src/bartender/db/models/user.py | 47 ---- src/bartender/settings.py | 26 +- src/bartender/web/api/applications/views.py | 5 +- src/bartender/web/api/job/views.py | 3 +- src/bartender/web/api/role/__init__.py | 4 - src/bartender/web/api/role/views.py | 109 --------- src/bartender/web/api/router.py | 4 +- src/bartender/web/api/user/__init__.py | 4 - src/bartender/web/api/user/schema.py | 41 ---- src/bartender/web/api/user/views.py | 53 ----- src/bartender/web/application.py | 3 - src/bartender/web/users.py | 70 ++++++ src/bartender/web/users/__init__.py | 1 - src/bartender/web/users/manager.py | 225 ------------------ src/bartender/web/users/orcid.py | 73 ------ src/bartender/web/users/router.py | 127 ---------- src/bartender/web/users/schema.py | 28 --- 28 files changed, 179 insertions(+), 936 deletions(-) create mode 100644 dummy_token_generator.py delete mode 100644 src/bartender/db/dao/user_dao.py delete mode 100644 src/bartender/db/models/user.py delete mode 100644 src/bartender/web/api/role/__init__.py delete mode 100644 src/bartender/web/api/role/views.py delete mode 100644 src/bartender/web/api/user/__init__.py delete mode 100644 src/bartender/web/api/user/schema.py delete mode 100644 src/bartender/web/api/user/views.py create mode 100644 src/bartender/web/users.py delete mode 100644 src/bartender/web/users/__init__.py delete mode 100644 src/bartender/web/users/manager.py delete mode 100644 src/bartender/web/users/orcid.py delete mode 100644 src/bartender/web/users/router.py delete mode 100644 src/bartender/web/users/schema.py diff --git a/dummy_token_generator.py b/dummy_token_generator.py new file mode 100644 index 0000000..df60279 --- /dev/null +++ b/dummy_token_generator.py @@ -0,0 +1,77 @@ +from datetime import datetime, timedelta +from typing import Annotated, Sequence +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.backends import default_backend +from fastapi import Body, FastAPI +import jwt +from pydantic import BaseModel + +app = FastAPI() + + +def load_key_pair(): + """Key pair loader. + + ```shell + openssl genpkey -algorithm RSA -out private_key.pem -pkeyopt rsa_keygen_bits:2048 + openssl rsa -pubout -in private_key.pem -out public_key.pem + ``` + + Returns: + _description_ + """ + with open("private_key.pem", "rb") as key_file: + private_key = serialization.load_pem_private_key( + key_file.read(), + password=None, + backend=default_backend(), + ) + with open("public_key.pem", "rb") as key_file: + public_key = serialization.load_pem_public_key( + key_file.read(), + ) + return private_key, public_key + + +private_key, public_key = load_key_pair() + +# TODO add /.well-known/jwks.json endpoint +@app.get(".well-known/jwks.json") +async def root(): + # TODO find library to generate jwks + return { + "keys": [ + { + "kty": "RSA", + "alg": "RS256", + "use": "sig", + "n": public_key.public_numbers().n, + "kid": + } + ] + } + + +@app.get("/token") +def get_token( + username: str = "someone", +): + roles: Sequence[str] = ("expert", "guru") + expire = datetime.utcnow() + timedelta(minutes=15) + payload = { + "sub": username, + "exp": expire, + "roles": roles, + } + return jwt.encode(payload, private_key, algorithm="RS256") + + +class Token(BaseModel): + jwt: str + + +@app.post("/verify") +def verify_token( + token: Token, +): + return jwt.decode(token.jwt, public_key, algorithms=["RS256"]) diff --git a/pyproject.toml b/pyproject.toml index e73272e..97468f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,10 +23,10 @@ alembic = "^1.10.2" asyncpg = "^0.27.0" python-multipart = "^0.0.6" aiofiles = "^23.1.0" -fastapi-users = {extras = ["oauth", "sqlalchemy"], version = "^10.4.1"} asyncssh = "^2.13.1" pyyaml = "^6.0" arq = "^0.25.0" +pyjwt = { extras=["crypto"], version = "^2.7.0"} [tool.poetry.group.dev.dependencies] pytest = "^7.0" diff --git a/src/bartender/__main__.py b/src/bartender/__main__.py index b8f4dbc..1556b7f 100644 --- a/src/bartender/__main__.py +++ b/src/bartender/__main__.py @@ -9,7 +9,6 @@ import uvicorn from bartender.config import build_config -from bartender.db.dao.user_dao import get_user_db from bartender.db.session import make_engine, make_session_factory from bartender.schedulers.arq import ArqSchedulerConfig, run_workers from bartender.settings import settings @@ -28,37 +27,6 @@ def serve() -> None: ) -async def make_super_async(email: str) -> None: - """Async method to grant a user super rights. - - Args: - email: Email of user - - Raises: - ValueError: When user can not be found - """ - session_factory = make_session_factory(make_engine()) - get_user_db_context = contextlib.asynccontextmanager(get_user_db) - async with session_factory() as session: - async with get_user_db_context(session) as user_db: - user = await user_db.get_by_email(email) - if user is None: - raise ValueError(f"User with {email} not found") - await user_db.give_super_powers(user) - print( # noqa: WPS421 -- user feedback on command line - f"User with {email} is now super user", - ) - - -def make_super(email: str) -> None: - """Grant a user super rights. - - Args: - email: Email of user - """ - asyncio.run(make_super_async(email)) - - def perform(config: Path, destination_names: Optional[list[str]] = None) -> None: """Runs arq worker to run queued jobs. @@ -102,10 +70,6 @@ def build_parser() -> ArgumentParser: serve_sp = subparsers.add_parser("serve", help="Serve web service") serve_sp.set_defaults(func=serve) - super_sp = subparsers.add_parser("super", help="Grant super rights to user") - super_sp.add_argument("email", help="Email address of logged in user") - super_sp.set_defaults(func=make_super) - perform_sp = subparsers.add_parser("perform", help="Async Redis queue job worker") perform_sp.add_argument( "--config", diff --git a/src/bartender/db/dao/job_dao.py b/src/bartender/db/dao/job_dao.py index 85d77c7..893b6e5 100644 --- a/src/bartender/db/dao/job_dao.py +++ b/src/bartender/db/dao/job_dao.py @@ -6,7 +6,6 @@ from bartender.db.dependencies import CurrentSession from bartender.db.models.job_model import Job, State -from bartender.db.models.user import User class JobDAO: @@ -19,7 +18,7 @@ async def create_job( # noqa: WPS211 self, name: Optional[str], application: str, - submitter: User, + submitter: str, updated_on: Optional[datetime] = None, created_on: Optional[datetime] = None, ) -> Optional[int]: @@ -48,7 +47,7 @@ async def create_job( # noqa: WPS211 await self.session.commit() return job.id - async def get_all_jobs(self, limit: int, offset: int, user: User) -> list[Job]: + async def get_all_jobs(self, limit: int, offset: int, user: str) -> list[Job]: """Get all job models of user with limit/offset pagination. Args: @@ -66,7 +65,7 @@ async def get_all_jobs(self, limit: int, offset: int, user: User) -> list[Job]: raw_jobs = await self.session.scalars(stmt) return list(raw_jobs.all()) - async def get_job(self, jobid: int, user: User) -> Job: + async def get_job(self, jobid: int, user: str) -> Job: """Get specific job model. Args: diff --git a/src/bartender/db/dao/user_dao.py b/src/bartender/db/dao/user_dao.py deleted file mode 100644 index 50b2eb2..0000000 --- a/src/bartender/db/dao/user_dao.py +++ /dev/null @@ -1,84 +0,0 @@ -from typing import Annotated, AsyncGenerator -from uuid import UUID - -from fastapi import Depends -from fastapi_users_db_sqlalchemy import SQLAlchemyUserDatabase -from sqlalchemy import select - -from bartender.db.dependencies import CurrentSession -from bartender.db.models.user import OAuthAccount, User - -# From app/db.py at -# https://fastapi-users.github.io/fastapi-users/10.1/configuration/full-example/ -# https://fastapi-users.github.io/fastapi-users/10.1/configuration/oauth/#sqlalchemy_1 - - -class UserDatabase(SQLAlchemyUserDatabase[User, UUID]): - """Class for accessing user tables. - - Extends fastapi_users.SQLAlchemyUserDatabase see - https://github.com/fastapi-users/fastapi-users-db-sqlalchemy/blob/main/fastapi_users_db_sqlalchemy/__init__.py#L96 - """ - - async def list(self, limit: int, offset: int) -> list[User]: - """Get list of users. - - Args: - limit: limit of users. - offset: offset of users. - - Returns: - list of users. - """ - statement = select(self.user_table).limit(limit).offset(offset) - results = await self.session.scalars(statement) - return list(results.unique().all()) - - async def assign_role(self, user: User, role: str) -> None: - """Assign a role to a user. - - Args: - user: The user. - role: The role. - """ - if role not in user.roles: - user.roles.append(role) - await self.session.commit() - await self.session.refresh(user) - - async def unassign_role(self, user: User, role: str) -> None: - """Unassign a role to a user. - - Args: - user: The user. - role: The role. - """ - if role in user.roles: - user.roles.remove(role) - await self.session.commit() - await self.session.refresh(user) - - async def give_super_powers(self, user: User) -> None: - """Give user super powers. - - Args: - user (User): The user. - """ - await self.update(user, {"is_superuser": True}) - - -async def get_user_db( - session: CurrentSession, -) -> AsyncGenerator[UserDatabase, None]: - """Factory method for accessing user table. - - Args: - session: SQLAlchemy session - - Yields: - Database adaptor - """ - yield UserDatabase(session, User, OAuthAccount) - - -CurrentUserDatabase = Annotated[UserDatabase, Depends(get_user_db)] diff --git a/src/bartender/db/migrations/versions/2022-08-30-15-39_300833a7a4c8.py b/src/bartender/db/migrations/versions/2022-08-30-15-39_300833a7a4c8.py index 5e0d055..6334072 100644 --- a/src/bartender/db/migrations/versions/2022-08-30-15-39_300833a7a4c8.py +++ b/src/bartender/db/migrations/versions/2022-08-30-15-39_300833a7a4c8.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa from alembic import op -from fastapi_users_db_sqlalchemy.generics import GUID # revision identifiers, used by Alembic. revision = "300833a7a4c8" @@ -18,22 +17,11 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "user", - sa.Column("email", sa.String(length=320), nullable=False), - sa.Column("hashed_password", sa.String(length=1024), nullable=False), - sa.Column("is_active", sa.Boolean(), nullable=False), - sa.Column("is_superuser", sa.Boolean(), nullable=False), - sa.Column("is_verified", sa.Boolean(), nullable=False), - sa.Column("id", GUID(), nullable=False), - sa.PrimaryKeyConstraint("id"), - ) - op.create_index(op.f("ix_user_email"), "user", ["email"], unique=True) + pass # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_index(op.f("ix_user_email"), table_name="user") - op.drop_table("user") + pass # ### end Alembic commands ### diff --git a/src/bartender/db/migrations/versions/2022-08-30-17-20_ef872771a841.py b/src/bartender/db/migrations/versions/2022-08-30-17-20_ef872771a841.py index ade3a87..e53144d 100644 --- a/src/bartender/db/migrations/versions/2022-08-30-17-20_ef872771a841.py +++ b/src/bartender/db/migrations/versions/2022-08-30-17-20_ef872771a841.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa from alembic import op -from fastapi_users_db_sqlalchemy.generics import GUID # revision identifiers, used by Alembic. revision = "ef872771a841" @@ -18,37 +17,11 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "oauth_account", - sa.Column("oauth_name", sa.String(length=100), nullable=False), - sa.Column("access_token", sa.String(length=1024), nullable=False), - sa.Column("expires_at", sa.BigInteger(), nullable=True), - sa.Column("refresh_token", sa.String(length=1024), nullable=True), - sa.Column("account_id", sa.String(length=320), nullable=False), - sa.Column("account_email", sa.String(length=320), nullable=False), - sa.Column("id", GUID(), nullable=False), - sa.Column("user_id", GUID(), nullable=False), - sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="cascade"), - sa.PrimaryKeyConstraint("id"), - ) - op.create_index( - op.f("ix_oauth_account_account_id"), - "oauth_account", - ["account_id"], - unique=False, - ) - op.create_index( - op.f("ix_oauth_account_oauth_name"), - "oauth_account", - ["oauth_name"], - unique=False, - ) + pass # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_index(op.f("ix_oauth_account_oauth_name"), table_name="oauth_account") - op.drop_index(op.f("ix_oauth_account_account_id"), table_name="oauth_account") - op.drop_table("oauth_account") + pass # ### end Alembic commands ### diff --git a/src/bartender/db/migrations/versions/2022-09-09-14-54_a31176b2cffa.py b/src/bartender/db/migrations/versions/2022-09-09-14-54_a31176b2cffa.py index fec36da..565efbc 100644 --- a/src/bartender/db/migrations/versions/2022-09-09-14-54_a31176b2cffa.py +++ b/src/bartender/db/migrations/versions/2022-09-09-14-54_a31176b2cffa.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa from alembic import op -from fastapi_users_db_sqlalchemy.generics import GUID # revision identifiers, used by Alembic. revision = "a31176b2cffa" @@ -27,7 +26,7 @@ def upgrade() -> None: "job", sa.Column( "submitter_id", - GUID(), + sa.String(length=200), nullable=False, ), ) @@ -39,13 +38,11 @@ def upgrade() -> None: "job", sa.Column("updated_on", sa.DateTime(timezone=True), nullable=False), ) - op.create_foreign_key("job_submitter_fk", "job", "user", ["submitter_id"], ["id"]) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_constraint("job_submitter_fk", "job", type_="foreignkey") op.drop_column("job", "updated_on") op.drop_column("job", "created_on") op.drop_column("job", "submitter_id") diff --git a/src/bartender/db/migrations/versions/2023-03-02-08-12_3226c12cc7f8.py b/src/bartender/db/migrations/versions/2023-03-02-08-12_3226c12cc7f8.py index 976816b..97a6942 100644 --- a/src/bartender/db/migrations/versions/2023-03-02-08-12_3226c12cc7f8.py +++ b/src/bartender/db/migrations/versions/2023-03-02-08-12_3226c12cc7f8.py @@ -18,18 +18,11 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.add_column( - "user", - sa.Column( - "roles", - postgresql.ARRAY(sa.String(length=100), dimensions=1), - default=[], - ), - ) + pass # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_column("user", "roles") + pass # ### end Alembic commands ### diff --git a/src/bartender/db/migrations/versions/2023-04-03-16-51_cf2424f395bc.py b/src/bartender/db/migrations/versions/2023-04-03-16-51_cf2424f395bc.py index af79132..3296040 100644 --- a/src/bartender/db/migrations/versions/2023-04-03-16-51_cf2424f395bc.py +++ b/src/bartender/db/migrations/versions/2023-04-03-16-51_cf2424f395bc.py @@ -19,22 +19,10 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.alter_column("job", "name", existing_type=sa.VARCHAR(length=200), nullable=False) - op.alter_column( - "user", - "roles", - existing_type=postgresql.ARRAY(sa.VARCHAR(length=100)), - nullable=False, - ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.alter_column( - "user", - "roles", - existing_type=postgresql.ARRAY(sa.VARCHAR(length=100)), - nullable=True, - ) op.alter_column("job", "name", existing_type=sa.VARCHAR(length=200), nullable=True) # ### end Alembic commands ### diff --git a/src/bartender/db/models/job_model.py b/src/bartender/db/models/job_model.py index 8459868..c857e78 100644 --- a/src/bartender/db/models/job_model.py +++ b/src/bartender/db/models/job_model.py @@ -1,13 +1,10 @@ from datetime import datetime from typing import Literal, Optional -from fastapi_users_db_sqlalchemy.generics import GUID -from sqlalchemy.orm import Mapped, mapped_column, relationship -from sqlalchemy.sql.schema import ForeignKey +from sqlalchemy.orm import Mapped, mapped_column from sqlalchemy.sql.sqltypes import DateTime, String from bartender.db.base import Base -from bartender.db.models.user import User from bartender.db.utils import now State = Literal[ @@ -50,11 +47,7 @@ class Job(Base): String(length=20), # noqa: WPS432 default="new", ) - submitter_id: Mapped[GUID] = mapped_column( - GUID(), - ForeignKey("user.id"), - ) - submitter: Mapped[User] = relationship(back_populates="jobs") + submitter: Mapped[str] = mapped_column(String(length=200)) # Identifier for job used by the scheduler internal_id: Mapped[Optional[str]] = mapped_column( String(length=200), # noqa: WPS432 diff --git a/src/bartender/db/models/user.py b/src/bartender/db/models/user.py deleted file mode 100644 index 801909f..0000000 --- a/src/bartender/db/models/user.py +++ /dev/null @@ -1,47 +0,0 @@ -from typing import TYPE_CHECKING, Optional - -from fastapi_users_db_sqlalchemy import ( - SQLAlchemyBaseOAuthAccountTableUUID, - SQLAlchemyBaseUserTableUUID, -) -from sqlalchemy import BigInteger -from sqlalchemy.dialects.postgresql import ARRAY -from sqlalchemy.ext.mutable import MutableList -from sqlalchemy.orm import Mapped, mapped_column, relationship -from sqlalchemy.sql.sqltypes import String - -from bartender.db.base import Base - -# From app/db.py at -# https://fastapi-users.github.io/fastapi-users/10.1/configuration/full-example/ -# https://fastapi-users.github.io/fastapi-users/10.1/configuration/oauth/#sqlalchemy_1 - - -class OAuthAccount(SQLAlchemyBaseOAuthAccountTableUUID, Base): - """Model for the social OAuth accounts.""" - - if TYPE_CHECKING: # pragma: no cover # noqa: WPS604 -- Python and mypy like it - expires_at: Optional[int] - else: - # Orcid returns expire of 2293079986 which is greater then Integer|int32 - expires_at: Mapped[Optional[int]] = mapped_column(BigInteger, nullable=True) - - -if TYPE_CHECKING: - from bartender.db.models.job_model import Job - - -class User(SQLAlchemyBaseUserTableUUID, Base): - """Model for the User.""" - - oauth_accounts: Mapped[list[OAuthAccount]] = relationship(lazy="joined") - jobs: Mapped[list["Job"]] = relationship( - back_populates="submitter", - ) - roles: Mapped[list[str]] = mapped_column( - MutableList.as_mutable(ARRAY(String(100), dimensions=1)), - default=[], - ) - - def __repr__(self) -> str: - return f"" diff --git a/src/bartender/settings.py b/src/bartender/settings.py index 3288a84..cfebc1b 100644 --- a/src/bartender/settings.py +++ b/src/bartender/settings.py @@ -1,8 +1,9 @@ import logging from pathlib import Path from tempfile import gettempdir -from typing import Literal, Optional +from typing import Literal +from cryptography.hazmat.primitives import serialization from pydantic import BaseSettings, Field from pydantic.types import FilePath from yarl import URL @@ -70,17 +71,7 @@ class Settings(BaseSettings): # User auth secret: str = "SECRET" # TODO should not have default when running in production - # Social OAuth logins - # must set to non '' to have GitHub social login enabled - github_client_id: str = "" - github_client_secret: str = "" - github_redirect_url: Optional[str] = None - orcidsandbox_client_id: str = "" - orcidsandbox_client_secret: str = "" - orcidsandboxd_redirect_url: Optional[str] = None - orcid_client_id: str = "" - orcid_client_secret: str = "" - orcid_redirect_url: Optional[str] = None + jwt_public_key_path: Path = Path("public_key.pem") # Settings for configuration config_filename: FilePath = Field(default_factory=default_config_filename) @@ -101,6 +92,17 @@ def db_url(self) -> URL: path=f"/{self.db_base}", ) + @property + def jwt_public_key(self): + # TODO read public key from JWKS endpoint + # TODO public key content as env variable + if not self.jwt_public_key_path.exists(): + with open(self.jwt_public_key_path, "rb") as key_file: + return serialization.load_pem_public_key( + key_file.read(), + ) + raise FileNotFoundError(self.jwt_public_key_path) + class Config: env_file = ".env" env_prefix = "BARTENDER_" diff --git a/src/bartender/web/api/applications/views.py b/src/bartender/web/api/applications/views.py index 1b2acca..c10b376 100644 --- a/src/bartender/web/api/applications/views.py +++ b/src/bartender/web/api/applications/views.py @@ -6,12 +6,11 @@ from bartender.config import ApplicatonConfiguration, CurrentConfig from bartender.context import Context, CurrentContext from bartender.db.dao.job_dao import CurrentJobDAO -from bartender.db.models.user import User from bartender.filesystem import has_config_file from bartender.filesystem.assemble_job import assemble_job from bartender.filesystem.stage_job_input import stage_job_input from bartender.web.api.applications.submit import submit -from bartender.web.users.manager import CurrentUser, current_api_token +from bartender.web.users import CurrentUser, User router = APIRouter() @@ -98,7 +97,7 @@ async def upload_job( # noqa: WPS211 job_dir = assemble_job( job_id, - await current_api_token(submitter), + submitter.apikey, context.job_root_dir, ) # TODO uploaded file can be big, and thus take long time to unpack, diff --git a/src/bartender/web/api/job/views.py b/src/bartender/web/api/job/views.py index 8673e70..63e2edb 100644 --- a/src/bartender/web/api/job/views.py +++ b/src/bartender/web/api/job/views.py @@ -14,7 +14,8 @@ from bartender.filesystems.queue import CurrentFileOutStagingQueue from bartender.web.api.job.schema import JobModelDTO from bartender.web.api.job.sync import sync_state, sync_states -from bartender.web.users.manager import CurrentUser +from bartender.web.users import CurrentUser + router = APIRouter() diff --git a/src/bartender/web/api/role/__init__.py b/src/bartender/web/api/role/__init__.py deleted file mode 100644 index 61fb2d6..0000000 --- a/src/bartender/web/api/role/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Job model API.""" -from bartender.web.api.role.views import router - -__all__ = ["router"] diff --git a/src/bartender/web/api/role/views.py b/src/bartender/web/api/role/views.py deleted file mode 100644 index a26c102..0000000 --- a/src/bartender/web/api/role/views.py +++ /dev/null @@ -1,109 +0,0 @@ -from uuid import UUID - -from fastapi import APIRouter, HTTPException -from starlette import status - -from bartender.config import CurrentRoles -from bartender.db.dao.user_dao import CurrentUserDatabase -from bartender.web.users.manager import CurrentSuperUser - -router = APIRouter() - - -@router.get("/") -async def list_roles( - roles: CurrentRoles, - super_user: CurrentSuperUser, -) -> list[str]: - """List available roles. - - Requires logged in user to be a super user. - - Args: - roles: Roles from config. - super_user: Checks if current user is super. - - Returns: - List of role names. - """ - return roles - - -@router.put("/{role_id}/{user_id}") -async def assign_role_to_user( - role_id: str, - user_id: str, - roles: CurrentRoles, - super_user: CurrentSuperUser, - user_db: CurrentUserDatabase, -) -> list[str]: - """Assign role to user. - - Requires super user powers. - - Args: - role_id: Role id - user_id: User id - roles: Set of allowed roles - super_user: Check if current user is super. - user_db: User db. - - Raises: - HTTPException: When user is not found - - Returns: - Roles assigned to user. - """ - user = await user_db.get(UUID(user_id)) - if user is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="User not found", - ) - if role_id not in roles: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Role not found", - ) - await user_db.assign_role(user, role_id) - return user.roles - - -@router.delete("/{role_id}/{user_id}") -async def unassign_role_from_user( - role_id: str, - user_id: str, - roles: CurrentRoles, - super_user: CurrentSuperUser, - user_db: CurrentUserDatabase, -) -> list[str]: - """Unassign role from user. - - Requires super user powers. - - Args: - role_id: Role id - user_id: User id - roles: Set of allowed roles - super_user: Check if current user is super. - user_db: User db. - - Raises: - HTTPException: When user is not found - - Returns: - Roles assigned to user. - """ - user = await user_db.get(UUID(user_id)) - if user is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="User not found", - ) - if role_id not in roles: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Role not found", - ) - await user_db.unassign_role(user, role_id) - return user.roles diff --git a/src/bartender/web/api/router.py b/src/bartender/web/api/router.py index 3ed4953..79ca26f 100644 --- a/src/bartender/web/api/router.py +++ b/src/bartender/web/api/router.py @@ -1,6 +1,6 @@ from fastapi.routing import APIRouter -from bartender.web.api import applications, job, monitoring, role, user +from bartender.web.api import applications, job, monitoring api_router = APIRouter() api_router.include_router(monitoring.router) @@ -10,5 +10,3 @@ prefix="/application", tags=["application"], ) -api_router.include_router(user.router, prefix="/users", tags=["users"]) -api_router.include_router(role.router, prefix="/roles", tags=["roles"]) diff --git a/src/bartender/web/api/user/__init__.py b/src/bartender/web/api/user/__init__.py deleted file mode 100644 index 0c77d7b..0000000 --- a/src/bartender/web/api/user/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Job model API.""" -from bartender.web.api.user.views import router - -__all__ = ["router"] diff --git a/src/bartender/web/api/user/schema.py b/src/bartender/web/api/user/schema.py deleted file mode 100644 index 835e608..0000000 --- a/src/bartender/web/api/user/schema.py +++ /dev/null @@ -1,41 +0,0 @@ -from typing import Any -from uuid import UUID - -from pydantic import BaseModel, validator - - -class OAuthAccountName(BaseModel): - """DTO for social provider name (OAuth account from GithHub or Orcid) of a user.""" - - oauth_name: str - account_id: str - account_email: str - - class Config: - orm_mode = True - - -class UserProfileInputDTO(BaseModel): - """DTO for profile of current user model.""" - - email: str - oauth_accounts: list[OAuthAccountName] - roles: list[str] - - @validator("roles", pre=True) - def _handle_roles_none(cls, value: Any) -> Any: # noqa: N805 is class method - if value is None: - return [] - return value - - class Config: - orm_mode = True - - -class UserAsListItem(UserProfileInputDTO): - """DTO for user in a list.""" - - id: UUID - is_active: bool - is_superuser: bool - is_verified: bool diff --git a/src/bartender/web/api/user/views.py b/src/bartender/web/api/user/views.py deleted file mode 100644 index 84255df..0000000 --- a/src/bartender/web/api/user/views.py +++ /dev/null @@ -1,53 +0,0 @@ -from fastapi import APIRouter - -from bartender.db.dao.user_dao import CurrentUserDatabase -from bartender.db.models.user import User -from bartender.web.api.user.schema import UserAsListItem, UserProfileInputDTO -from bartender.web.users.manager import CurrentSuperUser, CurrentUser - -router = APIRouter() - -# From app/app.py at -# https://fastapi-users.github.io/fastapi-users/10.1/configuration/full-example/ - - -@router.get("/profile", response_model=UserProfileInputDTO) -async def profile( - user: CurrentUser, -) -> User: - """ - Retrieve profile of currently logged in user. - - Args: - user: Current active user. - - Returns: - user profile. - """ - return user - - -@router.get( - "/", - response_model=list[UserAsListItem], -) -async def list_users( - super_user: CurrentSuperUser, - user_db: CurrentUserDatabase, - limit: int = 50, - offset: int = 0, -) -> list[User]: - """List of users. - - Requires super user powers. - - Args: - limit: Number of users to return. Defaults to 50. - offset: Offset. Defaults to 0. - super_user: Check if current user is super. - user_db: User db. - - Returns: - List of users. - """ - return await user_db.list(limit, offset) diff --git a/src/bartender/web/application.py b/src/bartender/web/application.py index 2935606..e6d396a 100644 --- a/src/bartender/web/application.py +++ b/src/bartender/web/application.py @@ -6,7 +6,6 @@ from bartender.web.api.router import api_router from bartender.web.lifespan import lifespan -from bartender.web.users.router import include_users_routes def get_app() -> FastAPI: @@ -31,8 +30,6 @@ def get_app() -> FastAPI: # Main router for the API. app.include_router(router=api_router, prefix="/api") - include_users_routes(app) - use_route_names_as_operation_ids(app) return app diff --git a/src/bartender/web/users.py b/src/bartender/web/users.py new file mode 100644 index 0000000..320cb1c --- /dev/null +++ b/src/bartender/web/users.py @@ -0,0 +1,70 @@ +from typing import Annotated, Optional, Sequence +from fastapi import Depends, HTTPException +import jwt +from pydantic import BaseModel +from bartender.settings import settings +from fastapi.security import HTTPBearer, APIKeyCookie, APIKeyQuery +from starlette.status import HTTP_403_FORBIDDEN + +header = HTTPBearer(bearerFormat='jwt', auto_error=False) +cookie = APIKeyCookie(name="bartenderToken", auto_error=False) +query = APIKeyQuery(name="token", auto_error=False) + + +class User(BaseModel): + username: str + roles: Sequence[str] + apikey: str + + +def current_api_token( + apikey: Annotated[Optional[str], Depends(header)], + apikey_from_cookie: Annotated[Optional[str], Depends(cookie)], + apikey_from_query: Annotated[Optional[str], Depends(query)], +): + if apikey_from_cookie: + return apikey_from_cookie + if apikey_from_query: + return apikey_from_query + if apikey: + return apikey + raise HTTPException(status_code=HTTP_403_FORBIDDEN, detail="Not authenticated") + + +def current_user( + apikey: Annotated[str, Depends(current_api_token)], +): + public_key = settings.jwt_public_key + # TODO catch exceptions and raise 40x error + data = jwt.decode( + apikey, + public_key, + algorithms=["RS256"], + # TODO verify more besides exp and public key + # like aud, iss, nbf + ) + return User( + username=data["sub"], + roles=data["roles"], + apikey=apikey, + ) + + +CurrentUser = Annotated[User, Depends(current_user)] + +# TODO add whoami endpoint which returns current user based on given token + +# TODO allow super user to read jobs from all users +# alternativly allow super user to impersonate other users +# super user is use with 'super' role in roles claims + +# TODO allow job to be readable by users who a member of a group +# 1. add endpoints to admin jobs groups +# 2. inside token of user add group memberships, +# use groups claims see https://www.iana.org/assignments/jwt/jwt.xhtml#claims + +# TODO allow job to be readable by anonymous users aka without token +# Used for storing example jobs or scenarios +# Public job should not expire +# 1. add endpoints to admin jobs public readability +# * endpoints should only be available to super user diff --git a/src/bartender/web/users/__init__.py b/src/bartender/web/users/__init__.py deleted file mode 100644 index a45e4a3..0000000 --- a/src/bartender/web/users/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Users & auth routes.""" diff --git a/src/bartender/web/users/manager.py b/src/bartender/web/users/manager.py deleted file mode 100644 index 4db3e96..0000000 --- a/src/bartender/web/users/manager.py +++ /dev/null @@ -1,225 +0,0 @@ -from typing import Annotated, Any, AsyncGenerator, Optional -from uuid import UUID - -from fastapi import Depends, Response -from fastapi.security import HTTPBearer -from fastapi_users import BaseUserManager, FastAPIUsers, UUIDIDMixin -from fastapi_users.authentication import ( - AuthenticationBackend, - BearerTransport, - JWTStrategy, -) -from fastapi_users.authentication.transport.base import ( - Transport, - TransportLogoutNotSupportedError, -) -from fastapi_users.authentication.transport.bearer import BearerResponse -from fastapi_users.jwt import generate_jwt -from fastapi_users.openapi import OpenAPIResponseType -from httpx_oauth.clients.github import GitHubOAuth2 -from starlette import status - -from bartender.db.dao.user_dao import CurrentUserDatabase -from bartender.db.models.user import User -from bartender.settings import settings -from bartender.web.users.orcid import OrcidOAuth2 - -# From app/users.py at -# https://fastapi-users.github.io/fastapi-users/10.1/configuration/full-example/ -# From app/users.py -# https://fastapi-users.github.io/fastapi-users/10.1/configuration/oauth/#sqlalchemy_1 - -github_oauth_client: Optional[GitHubOAuth2] = None -if settings.github_client_id != "": - github_oauth_client = GitHubOAuth2( - client_id=settings.github_client_id, - client_secret=settings.github_client_secret, - ) - -orcidsandbox_oauth_client: Optional[OrcidOAuth2] = None -if settings.orcidsandbox_client_id != "": - orcidsandbox_oauth_client = OrcidOAuth2( - client_id=settings.orcidsandbox_client_id, - client_secret=settings.orcidsandbox_client_secret, - is_sandbox=True, - ) - -orcid_oauth_client: Optional[OrcidOAuth2] = None -if settings.orcid_client_id != "": - orcid_oauth_client = OrcidOAuth2( - client_id=settings.orcid_client_id, - client_secret=settings.orcid_client_secret, - ) - - -class UserManager(UUIDIDMixin, BaseUserManager[User, UUID]): - """The user manager.""" - - reset_password_token_secret = settings.secret - verification_token_secret = settings.secret - - # For on_* methods see - # https://fastapi-users.github.io/fastapi-users/10.1/configuration/user-manager/ - - -async def get_user_manager( - user_db: CurrentUserDatabase, -) -> AsyncGenerator[UserManager, None]: - """Factory to get user manager. - - Args: - user_db: User database. - - Yields: - The manager. - """ - yield UserManager(user_db) - - -LIFETIME = 86400 # 24 hours - - -class JWTStrategyWithRoles(JWTStrategy[User, UUID]): - """JWT strategy with roles.""" - - async def write_token(self, user: User) -> str: - """Write token with user info. - - Args: - user: User from db - - Returns: - JWT token - """ - data = {"sub": str(user.id), "aud": self.token_audience, "roles": user.roles} - return generate_jwt( - data, - self.encode_key, - self.lifetime_seconds, - algorithm=self.algorithm, - ) - - -def get_jwt_strategy() -> JWTStrategy[User, UUID]: - """Get jwt strategy. - - Returns: - The strategy. - """ - return JWTStrategyWithRoles(secret=settings.secret, lifetime_seconds=LIFETIME) - - -local_auth_backend = AuthenticationBackend( - name="local", - transport=BearerTransport(tokenUrl="/auth/jwt/login"), - get_strategy=get_jwt_strategy, -) - - -class HTTPBearerTransport(Transport): - """After social login (Orcid, GitHub) you can use the JWT token to auth yourself.""" - - scheme: HTTPBearer - - def __init__(self) -> None: - self.scheme = HTTPBearer(bearerFormat="jwt", auto_error=False) - - async def get_login_response(self, token: str, response: Response) -> Any: - """Returns token after login. - - Args: - token: The token - response: The response - - Returns: - Token as JSON - """ - return BearerResponse(access_token=token, token_type="bearer") # noqa: S106 - - async def get_logout_response(self, response: Response) -> Any: - """Logout response. - - Args: - response: The response - - Raises: - TransportLogoutNotSupportedError: Always raises as JWT can not logout - """ - raise TransportLogoutNotSupportedError() - - @staticmethod - def get_openapi_login_responses_success() -> OpenAPIResponseType: - """Openapi response when login is success. - - Returns: - A reponse. - """ - return { - status.HTTP_200_OK: { - "model": BearerResponse, - "content": { - "application/json": { - "example": { - "access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1" - "c2VyX2lkIjoiOTIyMWZmYzktNjQwZi00MzcyLTg2Z" - "DMtY2U2NDJjYmE1NjAzIiwiYXVkIjoiZmFzdGFwaS" - "11c2VyczphdXRoIiwiZXhwIjoxNTcxNTA0MTkzfQ." - "M10bjOe45I5Ncu_uXvOmVV8QxnL-nZfcH96U90JaocI", - "token_type": "bearer", - }, - }, - }, - }, - } - - @staticmethod - def get_openapi_logout_responses_success() -> OpenAPIResponseType: - """Openapi response when logout is success. - - Returns: - A reponse. - """ - return {} - - -remote_auth_backend = AuthenticationBackend( - name="remote", - transport=HTTPBearerTransport(), - get_strategy=get_jwt_strategy, -) - -fastapi_users = FastAPIUsers[User, UUID]( - get_user_manager, - [local_auth_backend, remote_auth_backend], -) - -current_active_user = fastapi_users.current_user(active=True) - -CurrentUser = Annotated[User, Depends(current_active_user)] - -# TODO Token used by a job should be valid for as long as job can run. - -API_TOKEN_LIFETIME = 14400 # 4 hours - - -async def current_api_token(user: CurrentUser) -> str: - """Generate token that job can use to talk to bartender service. - - Args: - user: User that is currently logged in. - - Returns: - The token that can be put in HTTP header `Authorization: Bearer - `. - """ # noqa: DAR203 - # https://github.com/terrencepreilly/darglint/issues/53 - strategy: JWTStrategy[User, UUID] = JWTStrategy( - secret=settings.secret, - lifetime_seconds=API_TOKEN_LIFETIME, - ) - return await strategy.write_token(user) - - -current_super_user = fastapi_users.current_user(active=True, superuser=True) - -CurrentSuperUser = Annotated[User, Depends(current_super_user)] diff --git a/src/bartender/web/users/orcid.py b/src/bartender/web/users/orcid.py deleted file mode 100644 index eaf0618..0000000 --- a/src/bartender/web/users/orcid.py +++ /dev/null @@ -1,73 +0,0 @@ -from typing import Any, Dict, List, Optional, Tuple, cast - -from httpx_oauth.errors import GetIdEmailError -from httpx_oauth.oauth2 import BaseOAuth2 -from starlette import status - -AUTHORIZE_ENDPOINT = "https://{domain}/oauth/authorize" -ACCESS_TOKEN_ENDPOINT = "https://{domain}/oauth/token" # noqa: S105 -- not a password -BASE_SCOPES = ["openid"] -PROFILE_ENDPOINT = "https://{domain}/oauth/userinfo" -EMAILS_ENDPOINT = "https://pub.{domain}/v3.0/{id}/email" - - -class OrcidOAuth2(BaseOAuth2[Dict[str, Any]]): - """OAuth for Orcid.""" - - def __init__( - self, - client_id: str, - client_secret: str, - base_scopes: Optional[List[str]] = BASE_SCOPES, - is_sandbox: bool = False, - ): - self.domain = "sandbox.orcid.org" if is_sandbox else "orcid.org" - super().__init__( - client_id, - client_secret, - AUTHORIZE_ENDPOINT.format(domain=self.domain), - ACCESS_TOKEN_ENDPOINT.format(domain=self.domain), - name=self.domain, - base_scopes=base_scopes, - ) - - async def get_id_email(self, token: str) -> Tuple[str, str]: - """Retrieve account id and email. - - Args: - token: Orcid token - - Returns: - Tuple with account id and email - """ - orcid_id = await self._get_orcid_id(token) - email = await self._get_email(orcid_id) - return (orcid_id, email) - - async def _get_orcid_id(self, token: str) -> str: - async with self.get_httpx_client() as client: - headers = self.request_headers.copy() - headers["Authorization"] = f"Bearer {token}" - profile_response = await client.get( - PROFILE_ENDPOINT.format(domain=self.domain), - headers=headers, - ) - - if profile_response.status_code >= status.HTTP_400_BAD_REQUEST: - raise GetIdEmailError(profile_response.json()) - - profile_data = cast(Dict[str, str], profile_response.json()) - return profile_data["sub"] - - async def _get_email(self, orcid_id: str) -> str: - async with self.get_httpx_client() as client: - email_url = EMAILS_ENDPOINT.format(domain=self.domain, id=orcid_id) - email_response = await client.get(email_url, headers=self.request_headers) - - if email_response.status_code >= status.HTTP_400_BAD_REQUEST: - raise GetIdEmailError(email_response.json()) - - email_data = cast(Dict[str, Any], email_response.json()) - if "email" in email_data and email_data["email"]: - return cast(str, email_data["email"][0]["email"]) - return f"{orcid_id}@{self.domain}" diff --git a/src/bartender/web/users/router.py b/src/bartender/web/users/router.py deleted file mode 100644 index 6599f9c..0000000 --- a/src/bartender/web/users/router.py +++ /dev/null @@ -1,127 +0,0 @@ -from fastapi import FastAPI - -from bartender.settings import settings -from bartender.web.users.manager import ( - fastapi_users, - github_oauth_client, - local_auth_backend, - orcid_oauth_client, - orcidsandbox_oauth_client, - remote_auth_backend, -) -from bartender.web.users.schema import UserCreate, UserRead, UserUpdate - -# From app/app.py at -# https://fastapi-users.github.io/fastapi-users/10.1/configuration/full-example/ - - -def _github_routes(app: FastAPI) -> None: - # From app/app.py at - # https://fastapi-users.github.io/fastapi-users/10.1/configuration/oauth - if github_oauth_client is not None: - app.include_router( - fastapi_users.get_oauth_router( - github_oauth_client, - remote_auth_backend, - settings.secret, - associate_by_email=True, - redirect_url=settings.github_redirect_url, - ), - prefix="/auth/github", - tags=["auth"], - ) - app.include_router( - fastapi_users.get_oauth_associate_router( - github_oauth_client, - UserRead, - settings.secret, - ), - prefix="/auth/associate/github", - tags=["auth"], - ) - - -def _orcidsandbox_routes(app: FastAPI) -> None: - if orcidsandbox_oauth_client is not None: - app.include_router( - fastapi_users.get_oauth_router( - orcidsandbox_oauth_client, - remote_auth_backend, - settings.secret, - associate_by_email=True, - redirect_url=settings.orcidsandboxd_redirect_url, - ), - prefix="/auth/orcidsandbox", - tags=["auth"], - ) - app.include_router( - fastapi_users.get_oauth_associate_router( - orcidsandbox_oauth_client, - UserRead, - settings.secret, - ), - prefix="/auth/associate/orcidsandbox", - tags=["auth"], - ) - - -def _orcid_routes(app: FastAPI) -> None: - if orcid_oauth_client is not None: - app.include_router( - fastapi_users.get_oauth_router( - orcid_oauth_client, - remote_auth_backend, - settings.secret, - associate_by_email=True, - redirect_url=settings.orcid_redirect_url, - ), - prefix="/auth/orcid", - tags=["auth"], - ) - app.include_router( - fastapi_users.get_oauth_associate_router( - orcid_oauth_client, - UserRead, - settings.secret, - ), - prefix="/auth/associate/orcid", - tags=["auth"], - ) - - -def include_users_routes(app: FastAPI) -> None: - """Register fastapi_users routes. - - Args: - app: FastAPI app - """ - app.include_router( - fastapi_users.get_auth_router(local_auth_backend), - prefix="/auth/jwt", - tags=["auth"], - ) - app.include_router( - fastapi_users.get_register_router(UserRead, UserCreate), - prefix="/auth", - tags=["auth"], - ) - # Routes requiring sending mails have been removed - # as we prefer social login over local account. - # See https://fastapi-users.github.io/fastapi-users/10.1/configuration/full-example/ - # to add get_reset_password_router + get_verify_router back - - app.include_router( - fastapi_users.get_users_router(UserRead, UserUpdate), - prefix="/users", - tags=["users"], - ) - - # TODO For oauth logins an access token is given in the callback response. - # The Swagger UI does not allow to authorize with a access token. - # It would be nice to add JWT bearer security scheme to each protected route - # see https://spec.openapis.org/oas/v3.1.0#jwt-bearer-sample - # do this without breaking the username/password authorize in Swagger UI - - _github_routes(app) - _orcidsandbox_routes(app) - _orcid_routes(app) diff --git a/src/bartender/web/users/schema.py b/src/bartender/web/users/schema.py deleted file mode 100644 index 0b97ca1..0000000 --- a/src/bartender/web/users/schema.py +++ /dev/null @@ -1,28 +0,0 @@ -import uuid -from typing import Any - -from fastapi_users import schemas -from pydantic import validator - -# From app/schemas.py at -# https://fastapi-users.github.io/fastapi-users/10.1/configuration/full-example/ - - -class UserRead(schemas.BaseUser[uuid.UUID]): - """DTO for read user.""" - - roles: list[str] - - @validator("roles", pre=True) - def _handle_roles_none(cls, value: Any) -> Any: # noqa: N805 is class method - if value is None: - return [] - return value - - -class UserCreate(schemas.BaseUserCreate): - """DTO to create user.""" - - -class UserUpdate(schemas.BaseUserUpdate): - """DTO to update user.""" From c7ad0c6151035e89c5fd48a378f3f8d34f574391 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 18 Jul 2023 10:40:36 +0200 Subject: [PATCH 02/10] No need to store access token anymore --- .../versions/2023-06-16-12-39_02d0b8af4831.py | 33 ------------------- 1 file changed, 33 deletions(-) delete mode 100644 src/bartender/db/migrations/versions/2023-06-16-12-39_02d0b8af4831.py diff --git a/src/bartender/db/migrations/versions/2023-06-16-12-39_02d0b8af4831.py b/src/bartender/db/migrations/versions/2023-06-16-12-39_02d0b8af4831.py deleted file mode 100644 index 93b91b2..0000000 --- a/src/bartender/db/migrations/versions/2023-06-16-12-39_02d0b8af4831.py +++ /dev/null @@ -1,33 +0,0 @@ -"""egi access token >1024 chars - -Revision ID: 02d0b8af4831 -Revises: cf2424f395bc -Create Date: 2023-06-16 12:39:04.813291 - -""" -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision = "02d0b8af4831" -down_revision = "cf2424f395bc" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.alter_column( - "oauth_account", - "access_token", - type_=sa.VARCHAR(length=2048), - nullable=False, - ) - - -def downgrade() -> None: - op.alter_column( - "oauth_account", - "access_token", - type_=sa.VARCHAR(length=1048), - nullable=False, - ) From bc5dc59e03ba3b39441aba4bc3a91905a6c5a9e5 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 18 Jul 2023 12:22:47 +0200 Subject: [PATCH 03/10] Ping db in /api/health --- src/bartender/web/api/monitoring/views.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/bartender/web/api/monitoring/views.py b/src/bartender/web/api/monitoring/views.py index 6d0c5a8..45b779e 100644 --- a/src/bartender/web/api/monitoring/views.py +++ b/src/bartender/web/api/monitoring/views.py @@ -1,10 +1,15 @@ from fastapi import APIRouter +from sqlalchemy import text + +from bartender.db.dependencies import CurrentSession router = APIRouter() @router.get("/health") -def health_check() -> None: +async def health_check( + session: CurrentSession, +) -> None: """ Checks the health of a project. @@ -12,4 +17,5 @@ def health_check() -> None: """ # TODO check # 1. Database connection is live + await session.execute(text("SELECT 1")) # 2. Schedulers and filesystems of job destinations are working. From 79ac248fcbbcf54fa20bf9a249959ef0eb3aa70a Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 18 Jul 2023 15:58:56 +0200 Subject: [PATCH 04/10] Use private to generate token + public key to verify --- .gitignore | 5 +- README.md | 36 ++- docs/conf.py | 7 +- docs/configuration.md | 45 +++- docs/index.md | 1 - docs/user_management.md | 213 --------------- dummy_token_generator.py | 77 ------ poetry.lock | 250 +++++------------- pyproject.toml | 3 +- src/bartender/__main__.py | 154 ++++++++++- .../versions/2022-08-30-15-39_300833a7a4c8.py | 2 - .../versions/2022-08-30-17-20_ef872771a841.py | 2 - .../versions/2022-09-09-14-54_a31176b2cffa.py | 6 +- .../versions/2023-03-02-08-12_3226c12cc7f8.py | 3 - .../versions/2023-04-03-16-51_cf2424f395bc.py | 1 - src/bartender/db/models/job_model.py | 2 +- src/bartender/settings.py | 21 +- src/bartender/web/api/applications/views.py | 2 +- src/bartender/web/api/job/views.py | 5 +- src/bartender/web/api/monitoring/views.py | 9 +- src/bartender/web/api/router.py | 14 + src/bartender/web/users.py | 89 +++++-- src/bartender/web/users/egi.py | 36 --- 23 files changed, 391 insertions(+), 592 deletions(-) delete mode 100644 docs/user_management.md delete mode 100644 dummy_token_generator.py delete mode 100644 src/bartender/web/users/egi.py diff --git a/.gitignore b/.gitignore index d7b6a1c..d716811 100644 --- a/.gitignore +++ b/.gitignore @@ -142,5 +142,8 @@ dmypy.json # Cython debug symbols cython_debug/ -# Bartenders config file +# Bartender files config.yaml +jobs/ +private_key.pem +public_key.pem diff --git a/README.md b/README.md index 82499c7..1743b52 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,10 @@ Bartender is a middleware web service to schedule jobs on various infrastructures. It can run command line applications for visitors. The application input should -be a configuration file with links to data files in the same directory. After -authenticating with your local account or social login like ORCID or GitHub, you -can upload your configuration file and data files as an archive to the web -service for submission. Once the application has been executed the output files +be a configuration file with links to data files in the same directory. +After acquiring a JWT token, you can upload your configuration file and +data files as an archive to the web service for submission. +Once the job has been executed the output files can be browsed with a web browser. Bartender can be configured to run applications on a Slurm batch scheduler, @@ -27,6 +27,9 @@ Bartender can be used as the computational backend for a web application, the web application should guide visitors into the submission and show the results. See for an example. +Documentation for users and developers is available +at . + ## Quickstart 1. Install bartender via github: @@ -58,6 +61,16 @@ See for an example. alembic upgrade "head" ``` +1. Generate token to authenticate yourself + + ```bash + # Generate a rsa key pair + openssl genpkey -algorithm RSA -out private_key.pem \ + -pkeyopt rsa_keygen_bits:2048 + openssl rsa -pubout -in private_key.pem -out public_key.pem + bartender generate-token --username myname + ``` + 1. Run the application ```bash @@ -73,11 +86,16 @@ See for an example. The interactive API documentation generated by FastAPI is at -To consume the bartender web service you need to authenticate yourself. -Authentication is done by passing a JWT token in the HTTP header `Authorization: -Bearer ` in the HTTP request. This token can be aquired by using the -register+login routes or using a [socal -login](https://i-vresse-bartender.readthedocs.io/en/latest/user_management.html#user-management). +### Authentication + +To consume the bartender web service you need to authenticate yourself +with a [JWT token](https://jwt.io/) in the + +* HTTP header `Authorization: Bearer ` or +* query parameter `?token=` or +* Cookie `bartenderToken=`of the HTTP request. + +For more info see [Configuration docs](https://i-vresse-bartender.readthedocs.io/en/latest/configuration.html#authentication) ### Word count example diff --git a/docs/conf.py b/docs/conf.py index d47d4cd..720b4ca 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -39,11 +39,6 @@ intersphinx_mapping = { "python": ("https://docs.python.org/3", None), - # Commonly used libraries, uncomment when used in package - # 'numpy': ('http://docs.scipy.org/doc/numpy/', None), - # 'scipy': ('http://docs.scipy.org/doc/scipy/reference/', None), - # 'scikit-learn': ('https://scikit-learn.org/stable/', None), - # 'matplotlib': ('https://matplotlib.org/stable/', None), - # 'pandas': ('http://pandas.pydata.org/docs/', None), + "sqlalchemy": ("https://docs.sqlalchemy.org/en/14/", None), "asyncssh": ("https://asyncssh.readthedocs.io/en/latest/", None), } diff --git a/docs/configuration.md b/docs/configuration.md index 11ee9a4..a7616e0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -29,12 +29,46 @@ BARTENDER_ENVIRONMENT="dev" You can read more about BaseSettings class here: +## Authentication + +The bartender web service uses [JWT tokens](https://jwt.io/) for authentication. + +The tokens should use the RS256 algorithm, +which requires a public and private RSA key pair. +A key pair can be generated with + +```bash +openssl genpkey -algorithm RSA -out private_key.pem -pkeyopt rsa_keygen_bits:2048 +openssl rsa -pubout -in private_key.pem -out public_key.pem +``` + +The private key of the RSA key pair is used to generate a token in +an another web application or with the `bartender generate-token` command. + +The public key of the RSA key pair is used to verify that the token comes +from a trusted source. +The public key file location is `public_key.pem` +or value of `BARTENDER_PUBLIC_KEY` environment variable. + +The token payload should contain the following claims: + +* `sub`: The user id. Used to identifiy who submitted a job. +* `exp`: The expiration time of the token. +* `iss`: The issuer of the token. Used to track from where jobs are submitted. +* `roles`: The roles of the user. + See [Applications](#applications) how roles are used. + ## Configuration file -Bartender uses a configuration file for setting up applications and -destinations. An [example configuration -file](https://github.com/i-VRESSE/bartender/blob/main/config-example.yaml) is -shipped with the repository. Here, we explain the options in more detail. +Bartender uses a configuration file for setting up applications and destinations. + +The configuration file is `config.yaml` or +value of `BARTENDER_CONFIG_FILENAME` environment variable. +An +[example configuration file](https://github.com/i-VRESSE/bartender/blob/main/config-example.yaml) +is shipped with the repository. + +Here, we explain the options in more detail. ## Applications @@ -65,8 +99,7 @@ applications: replaced with value of the config key. * The `allowed_roles` key holds an array of role names, one of which a submitter should have. When key is not set or list is empty then any authorized user - is allowed. See [User management docs](user_management.md#roles) how to - assign/unassign roles to/from users. + is allowed. See [Authentication](#authentication) how to set roles on users. * The application command should not overwrite files uploaded during submission as these might not be downloaded from location where application is run. diff --git a/docs/index.md b/docs/index.md index a593eff..012bd72 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,6 +8,5 @@ self develop -user_management configuration ``` diff --git a/docs/user_management.md b/docs/user_management.md deleted file mode 100644 index fd0de07..0000000 --- a/docs/user_management.md +++ /dev/null @@ -1,213 +0,0 @@ -# User management - -For secure auth add `BARTENDER_SECRET=` to `.env` file. - -The web service can be configured to login with your social account for - -* GitHub -* Orcid -* EGI Check-in - -After you have setup a social login described in sub chapter below then you can -authenticate with - -```text -curl -X 'GET' \ - 'http://localhost:8000/auth//authorize' \ - -H 'accept: application/json' -``` - -This will return an authorization URL, which should be opened in web browser. - -Make sure the authorization URL and the callback URL configured in the social -platform have the same scheme, domain (like localhost or 127.0.0.1) and port. - -After visiting social authentication page you will get a JSON response with an -access token. - -This access token can be used on protected routes with - -```text -curl -X 'GET' \ - 'http://localhost:8000/api/users/profile' \ - -H 'accept: application/json' \ - -H 'Authorization: Bearer ' -``` - -## GitHub login - -The web service can be configured to login with your -[GitHub](https://gibhub.com) account. - -To enable perform following steps: - -1. Create a GitHub app - - 1. Goto - 2. Set Homepage URL to `http://localhost:8000/` - 3. Set Callback URL to `http://localhost:8000/auth/github/callback` - 4. Check `Request user authorization (OAuth) during installation` - 5. In Webhook section - - * Uncheck `Active` - - 6. In User permissions section - - * Set `Email addresses` to `Read-only` - - 7. Press `Create GitHub App` button - 8. After creation - - * Generate a new client secret - * (Optionally) Restrict app to certain IP addresses - -2. Append GitHub app credentials to `.env` file - - 1. Add `BARTENDER_GITHUB_CLIENT_ID=` - 2. Add `BARTENDER_GITHUB_CLIENT_SECRET=` - 3. (Optionally) Add URL of frontend server that captures token - `BARTENDER_GITHUB_REDIRECT_URL=` - -## Orcid sandbox login - -The web service can be configured to login with your [Orcid -sandbox](https://sandbox.orcid.org/) account. - -To enable perform following steps: - -1. Create Orcid account for yourself - - 1. Go to [https://sandbox.orcid.org/](https://sandbox.orcid.org/) - - Use `@mailinator.com` as email, because to register app you - need a verified email and Orcid sandbox only sends mails to - `mailinator.com`. - - 2. Go to - [https://www.mailinator.com/v4/public/inboxes.jsp](https://www.mailinator.com/v4/public/inboxes.jsp) - - Search for `` and verify your email address - - 3. Go to [https://sandbox.orcid.org/account](https://sandbox.orcid.org/account) - - Make email public for everyone - -2. Create application - - Goto - [https://sandbox.orcid.org/developer-tools](https://sandbox.orcid.org/developer-tools) - to register app. - - * Only one app can be registered per orcid account, so use alternate account - when primary account already has an registered app. - - * Your website URL does not allow localhost URL, so use - `https://github.com/i-VRESSE/bartender` - - * Redirect URI: for dev deployments set to - `http://localhost:8000/auth/orcidsandbox/callback` - -3. Append Orcid sandbox app credentials to `.env` file - - 1. Add `BARTENDER_ORCIDSANDBOX_CLIENT_ID=` - 2. Add `BARTENDER_ORCIDSANDBOX_CLIENT_SECRET=` - 3. (Optionally) Add URL of frontend server that captures token - `BARTENDER_ORCIDSANDBOX_REDIRECT_URL=` - -The `GET /api/users/profile` route will return the Orcid ID in -`oauth_accounts[oauth_name=sandbox.orcid.org].account_id`. - -## Orcid login - -The web service can be configured to login with your [Orcid](https://orcid.org/) -account. - -Steps are similar to [Orcid sandbox login](#orcid-sandbox-login), but - -* Callback URL must use **https** scheme -* Account emails don't have to be have be from `@mailinator.com` domain. -* In steps - - * Replace `https://sandbox.orcid.org/` with `https://orcid.org/` - * In redirect URL replace `orcidsandbox` with `orcid`. - * In `.env` replace `_ORCIDSANDBOX_` with `_ORCID_` - -## EGI Check-in login - -The web service can be configured to login with your [EGI Check-in](https://aai.egi.eu/) -account. - -To enable perform following steps: - -1. This web service needs to be [registered as a service provider in EGI Check-in](https://docs.egi.eu/providers/check-in/sp/). - * Select protocol: OIDC Service - * Callback should end with `/auth/egi/callback` - * Callback should for non-developement environments use https - * Disable PKCE, as the - [Python library](https://github.com/fastapi-users/fastapi-users) - used for authentication does support PKCE -2. Append EGI SP credentials to `.env` file - 1. Add `BARTENDER_EGI_CLIENT_ID=` - 2. Add `BARTENDER_EGI_CLIENT_SECRET=` - 3. (Optionally) Add which integration environment the SP is using, - `BARTENDER_EGI_ENVIRONMENT=` - 4. (Optionally) Add URL of frontend server that captures token - `BARTENDER_EGI_REDIRECT_URL=` - -## Super user - -When a user has `is_superuser is True` then he/she can manage users and make -other users also super users. - -However you need a first super user. This can be done by running - -```text -bartender super -``` - -## Roles - -An application can be configured to only allow users to submit jobs which -have a certain role. - -See [Configuration docs](configuration.md#applications) how to set allowed -roles on applications. - -To assign and unassign roles you will need to be a super user. - -Roles can be assigned to a user by calling - -```text -curl -X 'PUT' \ - 'http://localhost:8000/api/roles//' \ - -H 'accept: application/json' - -H 'Authorization: Bearer ' -``` - -Roles can be unassigned from a user by calling - -```text -curl -X 'DELETE' \ - 'http://localhost:8000/api/roles//' \ - -H 'accept: application/json' - -H 'Authorization: Bearer ' -``` - -The available roles can be found with - -```text -curl -X 'GET' \ - 'http://localhost:8000/api/roles/' \ - -H 'accept: application/json' - -H 'Authorization: Bearer ' -``` - -The id of a user can be found with - -```text -curl -X 'GET' \ - 'http://localhost:8000/api/users' \ - -H 'accept: application/json' - -H 'Authorization: Bearer ' -``` diff --git a/dummy_token_generator.py b/dummy_token_generator.py deleted file mode 100644 index df60279..0000000 --- a/dummy_token_generator.py +++ /dev/null @@ -1,77 +0,0 @@ -from datetime import datetime, timedelta -from typing import Annotated, Sequence -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.backends import default_backend -from fastapi import Body, FastAPI -import jwt -from pydantic import BaseModel - -app = FastAPI() - - -def load_key_pair(): - """Key pair loader. - - ```shell - openssl genpkey -algorithm RSA -out private_key.pem -pkeyopt rsa_keygen_bits:2048 - openssl rsa -pubout -in private_key.pem -out public_key.pem - ``` - - Returns: - _description_ - """ - with open("private_key.pem", "rb") as key_file: - private_key = serialization.load_pem_private_key( - key_file.read(), - password=None, - backend=default_backend(), - ) - with open("public_key.pem", "rb") as key_file: - public_key = serialization.load_pem_public_key( - key_file.read(), - ) - return private_key, public_key - - -private_key, public_key = load_key_pair() - -# TODO add /.well-known/jwks.json endpoint -@app.get(".well-known/jwks.json") -async def root(): - # TODO find library to generate jwks - return { - "keys": [ - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "n": public_key.public_numbers().n, - "kid": - } - ] - } - - -@app.get("/token") -def get_token( - username: str = "someone", -): - roles: Sequence[str] = ("expert", "guru") - expire = datetime.utcnow() + timedelta(minutes=15) - payload = { - "sub": username, - "exp": expire, - "roles": roles, - } - return jwt.encode(payload, private_key, algorithm="RS256") - - -class Token(BaseModel): - jwt: str - - -@app.post("/verify") -def verify_token( - token: Token, -): - return jwt.decode(token.jwt, public_key, algorithms=["RS256"]) diff --git a/poetry.lock b/poetry.lock index 0daccab..791f53b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -288,41 +288,6 @@ test = ["beautifulsoup4 (>=4.8.0)", "coverage (>=4.5.4)", "fixtures (>=3.0.0)", toml = ["tomli (>=1.1.0)"] yaml = ["PyYAML"] -[[package]] -name = "bcrypt" -version = "4.0.1" -description = "Modern password hashing for your software and your servers" -category = "main" -optional = false -python-versions = ">=3.6" -files = [ - {file = "bcrypt-4.0.1-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:b1023030aec778185a6c16cf70f359cbb6e0c289fd564a7cfa29e727a1c38f8f"}, - {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:08d2947c490093a11416df18043c27abe3921558d2c03e2076ccb28a116cb6d0"}, - {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0eaa47d4661c326bfc9d08d16debbc4edf78778e6aaba29c1bc7ce67214d4410"}, - {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae88eca3024bb34bb3430f964beab71226e761f51b912de5133470b649d82344"}, - {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:a522427293d77e1c29e303fc282e2d71864579527a04ddcfda6d4f8396c6c36a"}, - {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:fbdaec13c5105f0c4e5c52614d04f0bca5f5af007910daa8b6b12095edaa67b3"}, - {file = "bcrypt-4.0.1-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ca3204d00d3cb2dfed07f2d74a25f12fc12f73e606fcaa6975d1f7ae69cacbb2"}, - {file = "bcrypt-4.0.1-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:089098effa1bc35dc055366740a067a2fc76987e8ec75349eb9484061c54f535"}, - {file = "bcrypt-4.0.1-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:e9a51bbfe7e9802b5f3508687758b564069ba937748ad7b9e890086290d2f79e"}, - {file = "bcrypt-4.0.1-cp36-abi3-win32.whl", hash = "sha256:2caffdae059e06ac23fce178d31b4a702f2a3264c20bfb5ff541b338194d8fab"}, - {file = "bcrypt-4.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:8a68f4341daf7522fe8d73874de8906f3a339048ba406be6ddc1b3ccb16fc0d9"}, - {file = "bcrypt-4.0.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf4fa8b2ca74381bb5442c089350f09a3f17797829d958fad058d6e44d9eb83c"}, - {file = "bcrypt-4.0.1-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:67a97e1c405b24f19d08890e7ae0c4f7ce1e56a712a016746c8b2d7732d65d4b"}, - {file = "bcrypt-4.0.1-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b3b85202d95dd568efcb35b53936c5e3b3600c7cdcc6115ba461df3a8e89f38d"}, - {file = "bcrypt-4.0.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cbb03eec97496166b704ed663a53680ab57c5084b2fc98ef23291987b525cb7d"}, - {file = "bcrypt-4.0.1-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:5ad4d32a28b80c5fa6671ccfb43676e8c1cc232887759d1cd7b6f56ea4355215"}, - {file = "bcrypt-4.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b57adba8a1444faf784394de3436233728a1ecaeb6e07e8c22c8848f179b893c"}, - {file = "bcrypt-4.0.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:705b2cea8a9ed3d55b4491887ceadb0106acf7c6387699fca771af56b1cdeeda"}, - {file = "bcrypt-4.0.1-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:2b3ac11cf45161628f1f3733263e63194f22664bf4d0c0f3ab34099c02134665"}, - {file = "bcrypt-4.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3100851841186c25f127731b9fa11909ab7b1df6fc4b9f8353f4f1fd952fbf71"}, - {file = "bcrypt-4.0.1.tar.gz", hash = "sha256:27d375903ac8261cfe4047f6709d16f7d18d39b1ec92aaf72af989552a650ebd"}, -] - -[package.extras] -tests = ["pytest (>=3.2.1,!=3.3.0)"] -typecheck = ["mypy"] - [[package]] name = "beautifulsoup4" version = "4.12.0" @@ -382,7 +347,7 @@ uvloop = ["uvloop (>=0.15.2)"] name = "certifi" version = "2022.12.7" description = "Python package for providing Mozilla's CA Bundle." -category = "main" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -483,7 +448,7 @@ files = [ name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" +category = "dev" optional = false python-versions = ">=3.7.0" files = [ @@ -739,27 +704,6 @@ files = [ {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, ] -[[package]] -name = "dnspython" -version = "2.3.0" -description = "DNS toolkit" -category = "main" -optional = false -python-versions = ">=3.7,<4.0" -files = [ - {file = "dnspython-2.3.0-py3-none-any.whl", hash = "sha256:89141536394f909066cabd112e3e1a37e4e654db00a25308b0f130bc3152eb46"}, - {file = "dnspython-2.3.0.tar.gz", hash = "sha256:224e32b03eb46be70e12ef6d64e0be123a64e621ab4c0822ff6d450d52a540b9"}, -] - -[package.extras] -curio = ["curio (>=1.2,<2.0)", "sniffio (>=1.1,<2.0)"] -dnssec = ["cryptography (>=2.6,<40.0)"] -doh = ["h2 (>=4.1.0)", "httpx (>=0.21.1)", "requests (>=2.23.0,<3.0.0)", "requests-toolbelt (>=0.9.1,<0.11.0)"] -doq = ["aioquic (>=0.9.20)"] -idna = ["idna (>=2.1,<4.0)"] -trio = ["trio (>=0.14,<0.23)"] -wmi = ["wmi (>=1.5.1,<2.0.0)"] - [[package]] name = "docker" version = "6.0.1" @@ -795,20 +739,23 @@ files = [ ] [[package]] -name = "email-validator" -version = "1.3.1" -description = "A robust email address syntax and deliverability validation library." +name = "ecdsa" +version = "0.18.0" +description = "ECDSA cryptographic signature library (pure python)" category = "main" optional = false -python-versions = ">=3.5" +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ - {file = "email_validator-1.3.1-py2.py3-none-any.whl", hash = "sha256:49a72f5fa6ed26be1c964f0567d931d10bf3fdeeacdf97bc26ef1cd2a44e0bda"}, - {file = "email_validator-1.3.1.tar.gz", hash = "sha256:d178c5c6fa6c6824e9b04f199cf23e79ac15756786573c190d2ad13089411ad2"}, + {file = "ecdsa-0.18.0-py2.py3-none-any.whl", hash = "sha256:80600258e7ed2f16b9aa1d7c295bd70194109ad5a30fdee0eaeefef1d4c559dd"}, + {file = "ecdsa-0.18.0.tar.gz", hash = "sha256:190348041559e21b22a1d65cee485282ca11a6f81d503fddb84d5017e9ed1e49"}, ] [package.dependencies] -dnspython = ">=1.15.0" -idna = ">=2.0.0" +six = ">=1.9.0" + +[package.extras] +gmpy = ["gmpy"] +gmpy2 = ["gmpy2"] [[package]] name = "eradicate" @@ -859,50 +806,6 @@ dev = ["pre-commit (>=2.17.0,<3.0.0)", "ruff (==0.0.138)", "uvicorn[standard] (> doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-markdownextradata-plugin (>=0.1.7,<0.3.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "pyyaml (>=5.3.1,<7.0.0)", "typer-cli (>=0.0.13,<0.0.14)", "typer[all] (>=0.6.1,<0.8.0)"] test = ["anyio[trio] (>=3.2.1,<4.0.0)", "black (==23.1.0)", "coverage[toml] (>=6.5.0,<8.0)", "databases[sqlite] (>=0.3.2,<0.7.0)", "email-validator (>=1.1.1,<2.0.0)", "flask (>=1.1.2,<3.0.0)", "httpx (>=0.23.0,<0.24.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.982)", "orjson (>=3.2.1,<4.0.0)", "passlib[bcrypt] (>=1.7.2,<2.0.0)", "peewee (>=3.13.3,<4.0.0)", "pytest (>=7.1.3,<8.0.0)", "python-jose[cryptography] (>=3.3.0,<4.0.0)", "python-multipart (>=0.0.5,<0.0.7)", "pyyaml (>=5.3.1,<7.0.0)", "ruff (==0.0.138)", "sqlalchemy (>=1.3.18,<1.4.43)", "types-orjson (==3.6.2)", "types-ujson (==5.7.0.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,<6.0.0)"] -[[package]] -name = "fastapi-users" -version = "10.4.1" -description = "Ready-to-use and customizable users management for FastAPI" -category = "main" -optional = false -python-versions = ">=3.7" -files = [ - {file = "fastapi_users-10.4.1-py3-none-any.whl", hash = "sha256:42e24cd7dab59989c40b4beae970292d9a380ada234334d9d656488e533adc0f"}, - {file = "fastapi_users-10.4.1.tar.gz", hash = "sha256:cbc589278ce35b4e6218ec00987c64bb4dfbfbde39c192d8327b51eda171b9f1"}, -] - -[package.dependencies] -email-validator = ">=1.1.0,<1.4" -fastapi = ">=0.65.2" -fastapi-users-db-sqlalchemy = {version = ">=4.0.0", optional = true, markers = "extra == \"sqlalchemy\""} -httpx-oauth = {version = ">=0.4,<0.12", optional = true, markers = "extra == \"oauth\""} -makefun = ">=1.11.2,<2.0.0" -passlib = {version = "1.7.4", extras = ["bcrypt"]} -pyjwt = {version = "2.6.0", extras = ["crypto"]} -python-multipart = "0.0.6" - -[package.extras] -beanie = ["fastapi-users-db-beanie (>=1.0.0)"] -oauth = ["httpx-oauth (>=0.4,<0.12)"] -redis = ["redis (>=4.3.3,<5.0.0)"] -sqlalchemy = ["fastapi-users-db-sqlalchemy (>=4.0.0)"] - -[[package]] -name = "fastapi-users-db-sqlalchemy" -version = "5.0.0" -description = "FastAPI Users database adapter for SQLAlchemy" -category = "main" -optional = false -python-versions = ">=3.7" -files = [ - {file = "fastapi_users_db_sqlalchemy-5.0.0-py3-none-any.whl", hash = "sha256:7c9965555e94335d432f82f555b523809e1c37ed3ccd6ee1c7c9c0ae3240ea85"}, - {file = "fastapi_users_db_sqlalchemy-5.0.0.tar.gz", hash = "sha256:013c2800ce1cb5149acbd1d7eb8596e4aa7dd578395c40df951b666a121388c5"}, -] - -[package.dependencies] -fastapi-users = ">=10.0.0" -sqlalchemy = {version = ">=2.0.0,<2.1.0", extras = ["asyncio"]} - [[package]] name = "filelock" version = "3.10.7" @@ -1404,7 +1307,7 @@ files = [ name = "httpcore" version = "0.14.7" description = "A minimal low-level HTTP client." -category = "main" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1480,7 +1383,7 @@ test = ["Cython (>=0.29.24,<0.30.0)"] name = "httpx" version = "0.22.0" description = "The next generation HTTP client." -category = "main" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1501,21 +1404,6 @@ cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10.0.0,<1 http2 = ["h2 (>=3,<5)"] socks = ["socksio (>=1.0.0,<2.0.0)"] -[[package]] -name = "httpx-oauth" -version = "0.11.2" -description = "Async OAuth client using HTTPX" -category = "main" -optional = false -python-versions = ">=3.7" -files = [ - {file = "httpx_oauth-0.11.2-py3-none-any.whl", hash = "sha256:11a2c79111e61a7bd70624606ad6a3013b6756ce4817dd5d5b44f70bacb45940"}, - {file = "httpx_oauth-0.11.2.tar.gz", hash = "sha256:690e4d0a03b1974b9e600436a6d7a8c064e0707a3d86a41a12153cc797244ac6"}, -] - -[package.dependencies] -httpx = ">=0.18,<0.24" - [[package]] name = "identify" version = "2.5.22" @@ -1669,18 +1557,6 @@ files = [ {file = "lazy_object_proxy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:db1c1722726f47e10e0b5fdbf15ac3b8adb58c091d12b3ab713965795036985f"}, ] -[[package]] -name = "makefun" -version = "1.15.1" -description = "Small library to dynamically create python functions." -category = "main" -optional = false -python-versions = "*" -files = [ - {file = "makefun-1.15.1-py2.py3-none-any.whl", hash = "sha256:a63cfc7b47a539c76d97bd4fdb833c7d0461e759fd1225f580cb4be6200294d4"}, - {file = "makefun-1.15.1.tar.gz", hash = "sha256:40b0f118b6ded0d8d78c78f1eb679b8b6b2462e3c1b3e05fb1b2da8cd46b48a5"}, -] - [[package]] name = "mako" version = "1.2.4" @@ -2027,27 +1903,6 @@ files = [ {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, ] -[[package]] -name = "passlib" -version = "1.7.4" -description = "comprehensive password hashing framework supporting over 30 schemes" -category = "main" -optional = false -python-versions = "*" -files = [ - {file = "passlib-1.7.4-py2.py3-none-any.whl", hash = "sha256:aa6bca462b8d8bda89c70b382f0c298a20b5560af6cbfa2dce410c0a2fb669f1"}, - {file = "passlib-1.7.4.tar.gz", hash = "sha256:defd50f72b65c5402ab2c573830a6978e5f202ad0d984793c8dde2c4152ebe04"}, -] - -[package.dependencies] -bcrypt = {version = ">=3.1.0", optional = true, markers = "extra == \"bcrypt\""} - -[package.extras] -argon2 = ["argon2-cffi (>=18.2.0)"] -bcrypt = ["bcrypt (>=3.1.0)"] -build-docs = ["cloud-sptheme (>=1.10.1)", "sphinx (>=1.6)", "sphinxcontrib-fulltoc (>=1.2.0)"] -totp = ["cryptography"] - [[package]] name = "pathspec" version = "0.11.1" @@ -2139,6 +1994,18 @@ nodeenv = ">=0.11.1" pyyaml = ">=5.1" virtualenv = ">=20.10.0" +[[package]] +name = "pyasn1" +version = "0.5.0" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1-0.5.0-py2.py3-none-any.whl", hash = "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57"}, + {file = "pyasn1-0.5.0.tar.gz", hash = "sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde"}, +] + [[package]] name = "pycodestyle" version = "2.8.0" @@ -2262,27 +2129,6 @@ files = [ [package.extras] plugins = ["importlib-metadata"] -[[package]] -name = "pyjwt" -version = "2.6.0" -description = "JSON Web Token implementation in Python" -category = "main" -optional = false -python-versions = ">=3.7" -files = [ - {file = "PyJWT-2.6.0-py3-none-any.whl", hash = "sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14"}, - {file = "PyJWT-2.6.0.tar.gz", hash = "sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd"}, -] - -[package.dependencies] -cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} - -[package.extras] -crypto = ["cryptography (>=3.4.0)"] -dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] -docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] -tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] - [[package]] name = "pytest" version = "7.2.2" @@ -2355,6 +2201,29 @@ files = [ [package.extras] cli = ["click (>=5.0)"] +[[package]] +name = "python-jose" +version = "3.3.0" +description = "JOSE implementation in Python" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "python-jose-3.3.0.tar.gz", hash = "sha256:55779b5e6ad599c6336191246e95eb2293a9ddebd555f796a65f838f07e5d78a"}, + {file = "python_jose-3.3.0-py2.py3-none-any.whl", hash = "sha256:9b1376b023f8b298536eedd47ae1089bcdb848f1535ab30555cd92002d78923a"}, +] + +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"cryptography\""} +ecdsa = "!=0.15" +pyasn1 = "*" +rsa = "*" + +[package.extras] +cryptography = ["cryptography (>=3.4.0)"] +pycrypto = ["pyasn1", "pycrypto (>=2.6.0,<2.7.0)"] +pycryptodome = ["pyasn1", "pycryptodome (>=3.3.1,<4.0.0)"] + [[package]] name = "python-multipart" version = "0.0.6" @@ -2504,7 +2373,7 @@ docutils = ">=0.11,<1.0" name = "rfc3986" version = "1.5.0" description = "Validating URI References per RFC 3986" -category = "main" +category = "dev" optional = false python-versions = "*" files = [ @@ -2537,6 +2406,21 @@ pygments = ">=2.13.0,<3.0.0" [package.extras] jupyter = ["ipywidgets (>=7.5.1,<9)"] +[[package]] +name = "rsa" +version = "4.9" +description = "Pure-Python RSA implementation" +category = "main" +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, + {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" + [[package]] name = "setuptools" version = "67.6.1" @@ -3593,4 +3477,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "5648af52fc879c27c0a210ca0c319a35c793b8057fadacedd2851f4652a6dabe" +content-hash = "1703828417c658f2ad8e670399543a176780165d1b5984389748b0ebc1c98997" diff --git a/pyproject.toml b/pyproject.toml index 55e6cbb..82d5a53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ packages = [{include = "bartender", from = "src"}] [tool.poetry.dependencies] python = "^3.9" +# TODO upgrade to pydantic v2 fastapi = "^0.95.0" uvicorn = {version = "^0.21.1", extras = ["standard"]} pydantic = {version = "^1.10.7", extras = ["dotenv"]} @@ -26,8 +27,8 @@ aiofiles = "^23.1.0" asyncssh = "^2.13.1" pyyaml = "^6.0" arq = "^0.25.0" -pyjwt = { extras=["crypto"], version = "^2.7.0"} fs = "^2.4.16" +python-jose = {extras = ["cryptography"], version = "^3.3.0"} [tool.poetry.group.dev.dependencies] pytest = "^7.0" diff --git a/src/bartender/__main__.py b/src/bartender/__main__.py index 1556b7f..6ac5485 100644 --- a/src/bartender/__main__.py +++ b/src/bartender/__main__.py @@ -1,15 +1,20 @@ import asyncio -import contextlib import sys -from argparse import ArgumentParser +from argparse import ( + ArgumentDefaultsHelpFormatter, + ArgumentParser, + RawDescriptionHelpFormatter, +) +from datetime import datetime, timedelta from importlib.metadata import version from pathlib import Path -from typing import Optional +from textwrap import dedent +from typing import Any, Optional import uvicorn +from jose import jwt from bartender.config import build_config -from bartender.db.session import make_engine, make_session_factory from bartender.schedulers.arq import ArqSchedulerConfig, run_workers from bartender.settings import settings @@ -57,6 +62,66 @@ def perform(config: Path, destination_names: Optional[list[str]] = None) -> None asyncio.run(run_workers(configs)) +def generate_token( # noqa: WPS211 -- too many arguments + private_key: Path, + username: str, + roles: list[str], + lifetime: int, + issuer: str, + oformat: str, +) -> None: + """Generate a JSON Web Token (JWT) with the given parameters. + + Args: + private_key: Path to the private key file. + username: The username to include in the token. + roles: A list of roles to include in the token. + lifetime: The lifetime of the token in minutes. + issuer: The issuer of the token. + oformat: The format of the token output. Can be "header" or "string". + + Returns: + None + """ + # TODO use scope to allow different actions + # no scope could only be used to list applications and check health + # scope:read could be used to read your own job + # scope:write could be used to allow submission/deletion jobs + + # TODO allow super user to read jobs from all users + # by allowing super user to impersonate other users + # with act claim + # see https://www.rfc-editor.org/rfc/rfc8693.html#name-act-actor-claim + # https://auth0.com/docs/secure/tokens/json-web-tokens/json-web-token-claims + # https://www.iana.org/assignments/jwt/jwt.xhtml#claims + # alternativly a super user could also have 'super' role in roles claims + + # TODO allow job to be readable by users who is member of a group + # use groups claims see https://www.iana.org/assignments/jwt/jwt.xhtml#claims + # add group column job table, so on submission we store which group can read + # the job. Add endpoints to add/remove group to/from existing job + + # TODO allow job to be readable by anonymous users aka without token + # Used for storing example jobs or scenarios + # User should have super role. + # Add public boolena column to job table + # Add endpoints to make job public or private + # Public job should not expire + expire = datetime.utcnow() + timedelta(minutes=lifetime) + payload = { + "sub": username, + "exp": expire, + "roles": roles, + "iss": issuer, + } + private_key_body = Path(private_key).read_bytes() + token = jwt.encode(payload, private_key_body, algorithm="RS256") + if oformat == "header": + print(f"Authorization: Bearer {token}") # noqa: WPS421 -- user feedback + else: + print(token) # noqa: WPS421 -- user feedback + + def build_parser() -> ArgumentParser: """Build an argument parser. @@ -87,9 +152,90 @@ def build_parser() -> ArgumentParser: ) perform_sp.set_defaults(func=perform) + add_generate_token_subcommand(subparsers) + return parser +class Formatter(RawDescriptionHelpFormatter, ArgumentDefaultsHelpFormatter): + """Format help message for subcommands.""" + + pass # noqa: WPS420, WPS604 -- no need to implement methods + + +def add_generate_token_subcommand( + subparsers: Any, +) -> None: + """Add generate-token subcommand to parser. + + Args: + subparsers: Subparsers to add generate-token subcommand to. + """ + generate_token_sp = subparsers.add_parser( + "generate-token", + formatter_class=Formatter, + description=dedent( # noqa: WPS462 -- docs + """\ + Generate token. + + Token is required to consume the protected endpoints. + + Example: + ```shell + # Generate a rsa key pair + openssl genpkey -algorithm RSA -out private_key.pem \\ + -pkeyopt rsa_keygen_bits:2048 + openssl rsa -pubout -in private_key.pem -out public_key.pem + # Generate token + bartender generate-token --format header > token.txt + # Use token + curl -X 'GET' \\ + 'http://127.0.0.1:8000/api/whoami' \\ + -H 'accept: application/json' \\ + -H @token.txt | jq . + ``` + """, + ), + help="Generate token.", + ) + generate_token_sp.add_argument( + "--private-key", + default=Path("private_key.pem"), + type=Path, + help="Path to RSA private key file", + ) + generate_token_sp.add_argument( + "--username", + default="someone", + help="Username to use in token", + ) + generate_token_sp.add_argument( + "--roles", + nargs="+", + default=["expert", "guru"], + help="Roles to use in token", + ) + onehour_in_minutes = 60 + generate_token_sp.add_argument( + "--lifetime", + default=onehour_in_minutes, + type=int, + help="Lifetime of token in minutes", + ) + generate_token_sp.add_argument( + "--issuer", + default="bartendercli", + help="Issuer of token", + ) + generate_token_sp.add_argument( + "--oformat", + default="plain", + choices=["header", "plain"], + help="Format of output", + ) + generate_token_sp.set_defaults(func=generate_token) + + def main(argv: list[str] = sys.argv[1:]) -> None: """Entrypoint of the application. diff --git a/src/bartender/db/migrations/versions/2022-08-30-15-39_300833a7a4c8.py b/src/bartender/db/migrations/versions/2022-08-30-15-39_300833a7a4c8.py index 6334072..7970d35 100644 --- a/src/bartender/db/migrations/versions/2022-08-30-15-39_300833a7a4c8.py +++ b/src/bartender/db/migrations/versions/2022-08-30-15-39_300833a7a4c8.py @@ -5,8 +5,6 @@ Create Date: 2022-08-30 15:39:50.044644 """ -import sqlalchemy as sa -from alembic import op # revision identifiers, used by Alembic. revision = "300833a7a4c8" diff --git a/src/bartender/db/migrations/versions/2022-08-30-17-20_ef872771a841.py b/src/bartender/db/migrations/versions/2022-08-30-17-20_ef872771a841.py index e53144d..0a8ed35 100644 --- a/src/bartender/db/migrations/versions/2022-08-30-17-20_ef872771a841.py +++ b/src/bartender/db/migrations/versions/2022-08-30-17-20_ef872771a841.py @@ -5,8 +5,6 @@ Create Date: 2022-08-30 17:20:20.980685 """ -import sqlalchemy as sa -from alembic import op # revision identifiers, used by Alembic. revision = "ef872771a841" diff --git a/src/bartender/db/migrations/versions/2022-09-09-14-54_a31176b2cffa.py b/src/bartender/db/migrations/versions/2022-09-09-14-54_a31176b2cffa.py index 565efbc..3254478 100644 --- a/src/bartender/db/migrations/versions/2022-09-09-14-54_a31176b2cffa.py +++ b/src/bartender/db/migrations/versions/2022-09-09-14-54_a31176b2cffa.py @@ -25,8 +25,8 @@ def upgrade() -> None: op.add_column( "job", sa.Column( - "submitter_id", - sa.String(length=200), + "submitter", + sa.String(length=254), nullable=False, ), ) @@ -45,7 +45,7 @@ def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("job", "updated_on") op.drop_column("job", "created_on") - op.drop_column("job", "submitter_id") + op.drop_column("job", "submitter") op.drop_column("job", "state") op.drop_column("job", "application") # ### end Alembic commands ### diff --git a/src/bartender/db/migrations/versions/2023-03-02-08-12_3226c12cc7f8.py b/src/bartender/db/migrations/versions/2023-03-02-08-12_3226c12cc7f8.py index 97a6942..16b7afc 100644 --- a/src/bartender/db/migrations/versions/2023-03-02-08-12_3226c12cc7f8.py +++ b/src/bartender/db/migrations/versions/2023-03-02-08-12_3226c12cc7f8.py @@ -5,9 +5,6 @@ Create Date: 2023-03-02 08:12:57.458098 """ -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "3226c12cc7f8" diff --git a/src/bartender/db/migrations/versions/2023-04-03-16-51_cf2424f395bc.py b/src/bartender/db/migrations/versions/2023-04-03-16-51_cf2424f395bc.py index 3296040..07ef4e2 100644 --- a/src/bartender/db/migrations/versions/2023-04-03-16-51_cf2424f395bc.py +++ b/src/bartender/db/migrations/versions/2023-04-03-16-51_cf2424f395bc.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa from alembic import op -from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "cf2424f395bc" diff --git a/src/bartender/db/models/job_model.py b/src/bartender/db/models/job_model.py index c857e78..2116bf6 100644 --- a/src/bartender/db/models/job_model.py +++ b/src/bartender/db/models/job_model.py @@ -47,7 +47,7 @@ class Job(Base): String(length=20), # noqa: WPS432 default="new", ) - submitter: Mapped[str] = mapped_column(String(length=200)) + submitter: Mapped[str] = mapped_column(String(length=254)) # noqa: WPS432 # Identifier for job used by the scheduler internal_id: Mapped[Optional[str]] = mapped_column( String(length=200), # noqa: WPS432 diff --git a/src/bartender/settings.py b/src/bartender/settings.py index cfebc1b..803f41f 100644 --- a/src/bartender/settings.py +++ b/src/bartender/settings.py @@ -3,7 +3,8 @@ from tempfile import gettempdir from typing import Literal -from cryptography.hazmat.primitives import serialization +from jose import jwk +from jose.backends.base import Key from pydantic import BaseSettings, Field from pydantic.types import FilePath from yarl import URL @@ -71,7 +72,7 @@ class Settings(BaseSettings): # User auth secret: str = "SECRET" # TODO should not have default when running in production - jwt_public_key_path: Path = Path("public_key.pem") + public_key: FilePath = Path("public_key.pem") # Settings for configuration config_filename: FilePath = Field(default_factory=default_config_filename) @@ -84,6 +85,7 @@ def db_url(self) -> URL: database URL. """ return URL.build( + # TODO switch to sqlite so we don't need to run postgres container scheme="postgresql+asyncpg", host=self.db_host, port=self.db_port, @@ -93,15 +95,16 @@ def db_url(self) -> URL: ) @property - def jwt_public_key(self): + def jwt_key(self) -> Key: + """Public key object. + + Returns: + JOSE Key object for public key. + """ # TODO read public key from JWKS endpoint # TODO public key content as env variable - if not self.jwt_public_key_path.exists(): - with open(self.jwt_public_key_path, "rb") as key_file: - return serialization.load_pem_public_key( - key_file.read(), - ) - raise FileNotFoundError(self.jwt_public_key_path) + rsa_public_key = self.public_key.read_bytes() + return jwk.construct(rsa_public_key, "RS256") class Config: env_file = ".env" diff --git a/src/bartender/web/api/applications/views.py b/src/bartender/web/api/applications/views.py index c10b376..803b59c 100644 --- a/src/bartender/web/api/applications/views.py +++ b/src/bartender/web/api/applications/views.py @@ -91,7 +91,7 @@ async def upload_job( # noqa: WPS211 valid = context.applications.keys() raise KeyError(f"Invalid application. Valid applications: {valid}") _check_role(application, submitter, context) - job_id = await job_dao.create_job(upload.filename, application, submitter) + job_id = await job_dao.create_job(upload.filename, application, submitter.username) if job_id is None: raise IndexError("Failed to create database entry for job") diff --git a/src/bartender/web/api/job/views.py b/src/bartender/web/api/job/views.py index 778798b..7eac6c1 100644 --- a/src/bartender/web/api/job/views.py +++ b/src/bartender/web/api/job/views.py @@ -22,7 +22,6 @@ from bartender.web.api.job.sync import sync_state, sync_states from bartender.web.users import CurrentUser - router = APIRouter() @@ -53,7 +52,7 @@ async def retrieve_jobs( # noqa: WPS211 # TODO now list jobs that user submitted, # later also list jobs which are visible by admin # or are shared with current user - jobs = await job_dao.get_all_jobs(limit=limit, offset=offset, user=user) + jobs = await job_dao.get_all_jobs(limit=limit, offset=offset, user=user.username) # get current state for each job from scheduler await sync_states( jobs, @@ -95,7 +94,7 @@ async def retrieve_job( # or are shared with current user # TODO When job has state==ok then include URL to applications result page # TODO When job has state==error then include URL to error page - job = await job_dao.get_job(jobid=jobid, user=user) + job = await job_dao.get_job(jobid=jobid, user=user.username) if job.destination is not None: destination = context.destinations[job.destination] await sync_state( diff --git a/src/bartender/web/api/monitoring/views.py b/src/bartender/web/api/monitoring/views.py index 45b779e..c51f4ef 100644 --- a/src/bartender/web/api/monitoring/views.py +++ b/src/bartender/web/api/monitoring/views.py @@ -10,12 +10,13 @@ async def health_check( session: CurrentSession, ) -> None: - """ - Checks the health of a project. + """Checks the health of a project. It returns 200 if the project is healthy. + + Args: + session: SQLAlchemy session. """ - # TODO check - # 1. Database connection is live await session.execute(text("SELECT 1")) + # TODO check # 2. Schedulers and filesystems of job destinations are working. diff --git a/src/bartender/web/api/router.py b/src/bartender/web/api/router.py index 79ca26f..01d8261 100644 --- a/src/bartender/web/api/router.py +++ b/src/bartender/web/api/router.py @@ -1,6 +1,7 @@ from fastapi.routing import APIRouter from bartender.web.api import applications, job, monitoring +from bartender.web.users import CurrentUser, User api_router = APIRouter() api_router.include_router(monitoring.router) @@ -10,3 +11,16 @@ prefix="/application", tags=["application"], ) + + +@api_router.get("/whoami", tags=["user"]) +def whoami(user: CurrentUser) -> User: + """Get current user based on API key. + + Args: + user: Current user. + + Returns: + Current logged in user. + """ + return user diff --git a/src/bartender/web/users.py b/src/bartender/web/users.py index 320cb1c..469a9f7 100644 --- a/src/bartender/web/users.py +++ b/src/bartender/web/users.py @@ -1,70 +1,107 @@ from typing import Annotated, Optional, Sequence + from fastapi import Depends, HTTPException -import jwt +from fastapi.security import ( + APIKeyCookie, + APIKeyQuery, + HTTPAuthorizationCredentials, + HTTPBearer, +) +from jose import jwt from pydantic import BaseModel -from bartender.settings import settings -from fastapi.security import HTTPBearer, APIKeyCookie, APIKeyQuery from starlette.status import HTTP_403_FORBIDDEN -header = HTTPBearer(bearerFormat='jwt', auto_error=False) +from bartender.settings import settings + +header = HTTPBearer(bearerFormat="jwt", auto_error=False) cookie = APIKeyCookie(name="bartenderToken", auto_error=False) query = APIKeyQuery(name="token", auto_error=False) class User(BaseModel): + """User model.""" + username: str roles: Sequence[str] apikey: str def current_api_token( - apikey: Annotated[Optional[str], Depends(header)], + apikey: Annotated[Optional[HTTPAuthorizationCredentials], Depends(header)], apikey_from_cookie: Annotated[Optional[str], Depends(cookie)], apikey_from_query: Annotated[Optional[str], Depends(query)], -): +) -> str: + """Retrieve API token from header, cookie or query. + + Args: + apikey: API key from header + apikey_from_cookie: API key from cookie + apikey_from_query: API key from query + + Raises: + HTTPException: Forbidden 403 response if API key is not found. + + Returns: + API key + """ if apikey_from_cookie: + # Using api key inside cookie does not work with Swagger UI. + # however curl example works return apikey_from_cookie if apikey_from_query: return apikey_from_query if apikey: - return apikey + return apikey.credentials raise HTTPException(status_code=HTTP_403_FORBIDDEN, detail="Not authenticated") def current_user( apikey: Annotated[str, Depends(current_api_token)], -): - public_key = settings.jwt_public_key +) -> User: + """Current user based on API token. + + Args: + apikey: API key + + Returns: + User + """ + public_key = settings.jwt_key # TODO catch exceptions and raise 40x error + options = { + "verify_signature": True, + "verify_aud": True, + "verify_iat": True, + "verify_exp": True, + "verify_nbf": True, + "verify_iss": True, + "verify_sub": True, + "verify_jti": True, + "verify_at_hash": True, + "require_aud": False, + "require_iat": False, + "require_exp": True, + "require_nbf": False, + "require_iss": True, + "require_sub": True, + "require_jti": False, + "require_at_hash": False, + "leeway": 0, + } data = jwt.decode( apikey, public_key, algorithms=["RS256"], # TODO verify more besides exp and public key # like aud, iss, nbf + options=options, ) return User( username=data["sub"], roles=data["roles"], apikey=apikey, + # TODO store issuer in db so we can see from where job was submitted? ) CurrentUser = Annotated[User, Depends(current_user)] - -# TODO add whoami endpoint which returns current user based on given token - -# TODO allow super user to read jobs from all users -# alternativly allow super user to impersonate other users -# super user is use with 'super' role in roles claims - -# TODO allow job to be readable by users who a member of a group -# 1. add endpoints to admin jobs groups -# 2. inside token of user add group memberships, -# use groups claims see https://www.iana.org/assignments/jwt/jwt.xhtml#claims - -# TODO allow job to be readable by anonymous users aka without token -# Used for storing example jobs or scenarios -# Public job should not expire -# 1. add endpoints to admin jobs public readability -# * endpoints should only be available to super user diff --git a/src/bartender/web/users/egi.py b/src/bartender/web/users/egi.py deleted file mode 100644 index 3d4cc80..0000000 --- a/src/bartender/web/users/egi.py +++ /dev/null @@ -1,36 +0,0 @@ -from typing import List, Literal, Optional - -from httpx_oauth.clients.openid import OpenID - -# Scopes taken from https://aai.egi.eu/federation/egi/form/new -BASE_SCOPES = ["openid", "email", "profile", "voperson_id", "eduperson_entitlement"] -# From https://docs.egi.eu/providers/check-in/sp/#endpoints -CONFIGURATION_ENDPOINTS = { - "production": "https://aai.egi.eu/auth/realms/egi/.well-known/openid-configuration", - "development": "https://aai-dev.egi.eu/auth/realms/egi/.well-known/openid-configuration", # noqa: E501 - "demo": "https://aai-demo.egi.eu/auth/realms/egi/.well-known/openid-configuration", -} - - -class EgiCheckinOAuth2(OpenID): - """OAuth for EGI Check-in. - - See https://aai.egi.eu - - """ - - def __init__( - self, - client_id: str, - client_secret: str, - base_scopes: Optional[List[str]] = BASE_SCOPES, - environment: Literal["production", "development", "demo"] = "production", - ): - name = "EGI Check-in" - super().__init__( - client_id, - client_secret, - openid_configuration_endpoint=CONFIGURATION_ENDPOINTS[environment], - name=name, - base_scopes=base_scopes, - ) From d5b6ff8c03e1804a93be6b046aeca4bc3e9ac753 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 18 Jul 2023 16:27:10 +0200 Subject: [PATCH 05/10] Add submitting user to picker Fixes #25 --- docs/configuration.md | 1 + src/bartender/picker.py | 8 ++- src/bartender/user.py | 60 +++++++++++++++++++ src/bartender/web/api/applications/submit.py | 6 +- src/bartender/web/api/applications/views.py | 1 + src/bartender/web/users.py | 62 +++++--------------- tests/test_picker.py | 39 ++++++++---- 7 files changed, 116 insertions(+), 61 deletions(-) create mode 100644 src/bartender/user.py diff --git a/docs/configuration.md b/docs/configuration.md index a7616e0..5351256 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -270,6 +270,7 @@ The destination picker could look something like: def picker( job_dir: Path, application_name: str, + user: User, context: "Context", ) -> str: # Calculate size of job_dir in bytes diff --git a/src/bartender/picker.py b/src/bartender/picker.py index ca42388..2df9332 100644 --- a/src/bartender/picker.py +++ b/src/bartender/picker.py @@ -2,15 +2,18 @@ from pathlib import Path from typing import TYPE_CHECKING, Callable +from bartender.web.users import User + if TYPE_CHECKING: from bartender.context import Context -DestinationPicker = Callable[[Path, str, "Context"], str] +DestinationPicker = Callable[[Path, str, User, "Context"], str] def pick_first( job_dir: Path, application_name: str, + submitter: User, context: "Context", ) -> str: """Always picks first available destination from context. @@ -18,6 +21,7 @@ def pick_first( Args: job_dir: Location where job input files are located. application_name: Application name that should be run. + submitter: User that submitted the job. context: Context with applications and destinations. Returns: @@ -37,6 +41,7 @@ def __call__( self, job_dir: Path, application_name: str, + submitter: User, context: "Context", ) -> str: """Always picks the next destination. @@ -48,6 +53,7 @@ def __call__( Args: job_dir: Location where job input files are located. application_name: Application name that should be run. + submitter: User that submitted the job. context: Context with applications and destinations. Returns: diff --git a/src/bartender/user.py b/src/bartender/user.py new file mode 100644 index 0000000..570e92c --- /dev/null +++ b/src/bartender/user.py @@ -0,0 +1,60 @@ +from typing import Sequence + +from jose import jwt +from jose.backends.base import Key +from pydantic import BaseModel + + +class User(BaseModel): + """User model.""" + + username: str + roles: Sequence[str] = [] + apikey: str + + +def token2user(apikey: str, public_key: Key) -> User: + """Decodes a JWT token and returns a User object. + + Args: + apikey (str): The JWT token to decode. + public_key (Key): The public key to use for decoding the token. + + Returns: + User: A User object representing the decoded token. + """ + options = { + "verify_signature": True, + "verify_aud": True, + "verify_iat": True, + "verify_exp": True, + "verify_nbf": True, + "verify_iss": True, + "verify_sub": True, + "verify_jti": True, + "verify_at_hash": True, + "require_aud": False, + "require_iat": False, + "require_exp": True, + "require_nbf": False, + "require_iss": True, + "require_sub": True, + "require_jti": False, + "require_at_hash": False, + "leeway": 0, + } + data = jwt.decode( + apikey, + public_key, + algorithms=["RS256"], + # TODO verify more besides exp and public key + # like aud, iss, nbf + options=options, + # TODO check issuer is allowed with settings.issuer_whitelist + ) + return User( + username=data["sub"], + roles=data["roles"], + apikey=apikey, + # TODO store issuer in db so we can see from where job was submitted? + ) diff --git a/src/bartender/web/api/applications/submit.py b/src/bartender/web/api/applications/submit.py index 2447f0f..9ccc045 100644 --- a/src/bartender/web/api/applications/submit.py +++ b/src/bartender/web/api/applications/submit.py @@ -4,12 +4,14 @@ from bartender.db.dao.job_dao import JobDAO from bartender.filesystems.abstract import AbstractFileSystem from bartender.schedulers.abstract import JobDescription +from bartender.web.users import User -async def submit( +async def submit( # noqa: WPS211 external_job_id: int, job_dir: Path, application: str, + submitter: User, job_dao: JobDAO, context: Context, ) -> None: @@ -19,6 +21,7 @@ async def submit( external_job_id: External job id. job_dir: Location where job input files are located. application: Application name that should be run. + submitter: User that submitted the job. job_dao: JobDAO object. context: Context with applications and destinations. """ @@ -27,6 +30,7 @@ async def submit( destination_name = context.destination_picker( job_dir, application, + submitter, context, ) destination = context.destinations[destination_name] diff --git a/src/bartender/web/api/applications/views.py b/src/bartender/web/api/applications/views.py index 803b59c..c07c7e9 100644 --- a/src/bartender/web/api/applications/views.py +++ b/src/bartender/web/api/applications/views.py @@ -112,6 +112,7 @@ async def upload_job( # noqa: WPS211 job_id, job_dir, application, + submitter, job_dao, context, ) diff --git a/src/bartender/web/users.py b/src/bartender/web/users.py index 469a9f7..a7e359e 100644 --- a/src/bartender/web/users.py +++ b/src/bartender/web/users.py @@ -1,4 +1,4 @@ -from typing import Annotated, Optional, Sequence +from typing import Annotated, Optional from fastapi import Depends, HTTPException from fastapi.security import ( @@ -7,25 +7,17 @@ HTTPAuthorizationCredentials, HTTPBearer, ) -from jose import jwt -from pydantic import BaseModel -from starlette.status import HTTP_403_FORBIDDEN +from jose.exceptions import ExpiredSignatureError, JWTClaimsError, JWTError +from starlette.status import HTTP_401_UNAUTHORIZED, HTTP_403_FORBIDDEN from bartender.settings import settings +from bartender.user import User, token2user header = HTTPBearer(bearerFormat="jwt", auto_error=False) cookie = APIKeyCookie(name="bartenderToken", auto_error=False) query = APIKeyQuery(name="token", auto_error=False) -class User(BaseModel): - """User model.""" - - username: str - roles: Sequence[str] - apikey: str - - def current_api_token( apikey: Annotated[Optional[HTTPAuthorizationCredentials], Depends(header)], apikey_from_cookie: Annotated[Optional[str], Depends(cookie)], @@ -63,45 +55,21 @@ def current_user( Args: apikey: API key + Raises: + HTTPException: Unauthorized 401 response if API key is invalid. + Returns: User """ public_key = settings.jwt_key - # TODO catch exceptions and raise 40x error - options = { - "verify_signature": True, - "verify_aud": True, - "verify_iat": True, - "verify_exp": True, - "verify_nbf": True, - "verify_iss": True, - "verify_sub": True, - "verify_jti": True, - "verify_at_hash": True, - "require_aud": False, - "require_iat": False, - "require_exp": True, - "require_nbf": False, - "require_iss": True, - "require_sub": True, - "require_jti": False, - "require_at_hash": False, - "leeway": 0, - } - data = jwt.decode( - apikey, - public_key, - algorithms=["RS256"], - # TODO verify more besides exp and public key - # like aud, iss, nbf - options=options, - ) - return User( - username=data["sub"], - roles=data["roles"], - apikey=apikey, - # TODO store issuer in db so we can see from where job was submitted? - ) + try: + return token2user(apikey, public_key) + except ExpiredSignatureError as exception: + raise HTTPException(status_code=HTTP_401_UNAUTHORIZED, detail=str(exception)) + except JWTClaimsError as exception: + raise HTTPException(status_code=HTTP_401_UNAUTHORIZED, detail=str(exception)) + except JWTError as exception: + raise HTTPException(status_code=HTTP_401_UNAUTHORIZED, detail=str(exception)) CurrentUser = Annotated[User, Depends(current_user)] diff --git a/tests/test_picker.py b/tests/test_picker.py index 93f59ca..903502a 100644 --- a/tests/test_picker.py +++ b/tests/test_picker.py @@ -5,6 +5,15 @@ from bartender.context import ApplicatonConfiguration, Context from bartender.destinations import Destination from bartender.picker import PickRound, pick_first +from bartender.user import User + + +@pytest.fixture +def user() -> User: + return User( + username="user1", + apikey="apikey1", + ) class TestPickFirst: @@ -12,6 +21,7 @@ class TestPickFirst: async def test_with2destinations_returns_first( self, demo_destination: Destination, + user: User, ) -> None: context = Context( destination_picker=pick_first, @@ -24,13 +34,14 @@ async def test_with2destinations_returns_first( }, job_root_dir=Path("/jobs"), ) - actual = pick_first(context.job_root_dir / "job1", "app1", context) + + actual = pick_first(context.job_root_dir / "job1", "app1", user, context) expected = "d1" assert actual == expected @pytest.mark.anyio - async def test_nodestintations_returns_indexerror(self) -> None: + async def test_nodestintations_returns_indexerror(self, user: User) -> None: context = Context( destination_picker=pick_first, applications={ @@ -41,7 +52,7 @@ async def test_nodestintations_returns_indexerror(self) -> None: ) with pytest.raises(IndexError): - pick_first(context.job_root_dir / "job1", "app1", context) + pick_first(context.job_root_dir / "job1", "app1", user, context) class TestPickRoundWith2Destinations: @@ -60,33 +71,37 @@ async def context(self, demo_destination: Destination) -> Context: ) @pytest.mark.anyio - async def test_firstcall_returns_first(self, context: Context) -> None: + async def test_firstcall_returns_first(self, context: Context, user: User) -> None: picker = PickRound() - actual = picker(context.job_root_dir / "job1", "app1", context) + actual = picker(context.job_root_dir / "job1", "app1", user, context) expected = "d1" assert actual == expected @pytest.mark.anyio - async def test_secondcall_returns_second(self, context: Context) -> None: + async def test_secondcall_returns_second( + self, + context: Context, + user: User, + ) -> None: picker = PickRound() # first call - picker(context.job_root_dir / "job1", "app1", context) + picker(context.job_root_dir / "job1", "app1", user, context) # second call - actual = picker(context.job_root_dir / "job1", "app1", context) + actual = picker(context.job_root_dir / "job1", "app1", user, context) expected = "d2" assert actual == expected @pytest.mark.anyio - async def test_thirdcall_returns_first(self, context: Context) -> None: + async def test_thirdcall_returns_first(self, context: Context, user: User) -> None: picker = PickRound() # 1st call - picker(context.job_root_dir / "job1", "app1", context) + picker(context.job_root_dir / "job1", "app1", user, context) # 2nd call - picker(context.job_root_dir / "job1", "app1", context) + picker(context.job_root_dir / "job1", "app1", user, context) # 3rd call - actual = picker(context.job_root_dir / "job1", "app1", context) + actual = picker(context.job_root_dir / "job1", "app1", user, context) expected = "d1" assert actual == expected From a7b047f066bafc4d6f1b5193232476b298900438 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Wed, 19 Jul 2023 12:54:11 +0200 Subject: [PATCH 06/10] Fix tests + move jwt decoder creation from settings to lifespan + make public key optional but with warning --- poetry.lock | 2 +- pyproject.toml | 1 + src/bartender/__main__.py | 65 +---------- src/bartender/picker.py | 2 +- src/bartender/settings.py | 20 +--- src/bartender/user.py | 209 +++++++++++++++++++++++++++------- src/bartender/web/lifespan.py | 20 ++++ src/bartender/web/users.py | 38 ++++++- tests/conftest.py | 168 +++++++++------------------ tests/test_user.py | 31 +++++ tests/web/test_application.py | 27 ++--- tests/web/test_job.py | 6 +- tests/web/test_role.py | 144 ----------------------- tests/web/test_user.py | 94 --------------- 14 files changed, 323 insertions(+), 504 deletions(-) create mode 100644 tests/test_user.py delete mode 100644 tests/web/test_role.py delete mode 100644 tests/web/test_user.py diff --git a/poetry.lock b/poetry.lock index 791f53b..d8ff1a3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3477,4 +3477,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "1703828417c658f2ad8e670399543a176780165d1b5984389748b0ebc1c98997" +content-hash = "4311f5c79a1e34cdb7e4f5bfcd85a215e29200c5ab7e1c73359e22972e1a5aac" diff --git a/pyproject.toml b/pyproject.toml index 82d5a53..0577bbc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ httpx = "^0.22.0" types-aiofiles = "^23" testcontainers = "^3.7.0" types-pyyaml = "^6.0.12.2" +rsa = "^4.9" [tool.poetry.group.docs] optional = true diff --git a/src/bartender/__main__.py b/src/bartender/__main__.py index 6ac5485..7f07096 100644 --- a/src/bartender/__main__.py +++ b/src/bartender/__main__.py @@ -5,18 +5,17 @@ ArgumentParser, RawDescriptionHelpFormatter, ) -from datetime import datetime, timedelta from importlib.metadata import version from pathlib import Path from textwrap import dedent from typing import Any, Optional import uvicorn -from jose import jwt from bartender.config import build_config from bartender.schedulers.arq import ArqSchedulerConfig, run_workers from bartender.settings import settings +from bartender.user import generate_token_subcommand def serve() -> None: @@ -62,66 +61,6 @@ def perform(config: Path, destination_names: Optional[list[str]] = None) -> None asyncio.run(run_workers(configs)) -def generate_token( # noqa: WPS211 -- too many arguments - private_key: Path, - username: str, - roles: list[str], - lifetime: int, - issuer: str, - oformat: str, -) -> None: - """Generate a JSON Web Token (JWT) with the given parameters. - - Args: - private_key: Path to the private key file. - username: The username to include in the token. - roles: A list of roles to include in the token. - lifetime: The lifetime of the token in minutes. - issuer: The issuer of the token. - oformat: The format of the token output. Can be "header" or "string". - - Returns: - None - """ - # TODO use scope to allow different actions - # no scope could only be used to list applications and check health - # scope:read could be used to read your own job - # scope:write could be used to allow submission/deletion jobs - - # TODO allow super user to read jobs from all users - # by allowing super user to impersonate other users - # with act claim - # see https://www.rfc-editor.org/rfc/rfc8693.html#name-act-actor-claim - # https://auth0.com/docs/secure/tokens/json-web-tokens/json-web-token-claims - # https://www.iana.org/assignments/jwt/jwt.xhtml#claims - # alternativly a super user could also have 'super' role in roles claims - - # TODO allow job to be readable by users who is member of a group - # use groups claims see https://www.iana.org/assignments/jwt/jwt.xhtml#claims - # add group column job table, so on submission we store which group can read - # the job. Add endpoints to add/remove group to/from existing job - - # TODO allow job to be readable by anonymous users aka without token - # Used for storing example jobs or scenarios - # User should have super role. - # Add public boolena column to job table - # Add endpoints to make job public or private - # Public job should not expire - expire = datetime.utcnow() + timedelta(minutes=lifetime) - payload = { - "sub": username, - "exp": expire, - "roles": roles, - "iss": issuer, - } - private_key_body = Path(private_key).read_bytes() - token = jwt.encode(payload, private_key_body, algorithm="RS256") - if oformat == "header": - print(f"Authorization: Bearer {token}") # noqa: WPS421 -- user feedback - else: - print(token) # noqa: WPS421 -- user feedback - - def build_parser() -> ArgumentParser: """Build an argument parser. @@ -233,7 +172,7 @@ def add_generate_token_subcommand( choices=["header", "plain"], help="Format of output", ) - generate_token_sp.set_defaults(func=generate_token) + generate_token_sp.set_defaults(func=generate_token_subcommand) def main(argv: list[str] = sys.argv[1:]) -> None: diff --git a/src/bartender/picker.py b/src/bartender/picker.py index 2df9332..b46561f 100644 --- a/src/bartender/picker.py +++ b/src/bartender/picker.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Callable -from bartender.web.users import User +from bartender.user import User if TYPE_CHECKING: from bartender.context import Context diff --git a/src/bartender/settings.py b/src/bartender/settings.py index 803f41f..2949d18 100644 --- a/src/bartender/settings.py +++ b/src/bartender/settings.py @@ -3,8 +3,6 @@ from tempfile import gettempdir from typing import Literal -from jose import jwk -from jose.backends.base import Key from pydantic import BaseSettings, Field from pydantic.types import FilePath from yarl import URL @@ -69,10 +67,8 @@ class Settings(BaseSettings): db_base: str = "bartender" db_echo: bool = False - # User auth - secret: str = "SECRET" # TODO should not have default when running in production - - public_key: FilePath = Path("public_key.pem") + # RSA public key used to verify JWT tokens + public_key: Path = Path("public_key.pem") # Settings for configuration config_filename: FilePath = Field(default_factory=default_config_filename) @@ -94,18 +90,6 @@ def db_url(self) -> URL: path=f"/{self.db_base}", ) - @property - def jwt_key(self) -> Key: - """Public key object. - - Returns: - JOSE Key object for public key. - """ - # TODO read public key from JWKS endpoint - # TODO public key content as env variable - rsa_public_key = self.public_key.read_bytes() - return jwk.construct(rsa_public_key, "RS256") - class Config: env_file = ".env" env_prefix = "BARTENDER_" diff --git a/src/bartender/user.py b/src/bartender/user.py index 570e92c..3ebddcf 100644 --- a/src/bartender/user.py +++ b/src/bartender/user.py @@ -1,6 +1,8 @@ -from typing import Sequence +from datetime import datetime, timedelta +from pathlib import Path +from typing import Literal, Sequence -from jose import jwt +from jose import jwk, jwt from jose.backends.base import Key from pydantic import BaseModel @@ -13,48 +15,173 @@ class User(BaseModel): apikey: str -def token2user(apikey: str, public_key: Key) -> User: - """Decodes a JWT token and returns a User object. +class JwtDecoder: + """JWT decoder. Args: - apikey (str): The JWT token to decode. - public_key (Key): The public key to use for decoding the token. + key: The key to use for decoding the JWT token. + + """ + + def __init__(self, key: Key): + self.key = key + + def __call__(self, apikey: str) -> User: + """Decodes a JWT token and returns a User object. + + Raises an JOSE exception if the token is invalid. + + Args: + apikey (str): The JWT token to decode. + + Returns: + User: A User object. + """ + options = { + "verify_signature": True, + "verify_aud": True, + "verify_iat": True, + "verify_exp": True, + "verify_nbf": True, + "verify_iss": True, + "verify_sub": True, + "verify_jti": True, + "verify_at_hash": True, + "require_aud": False, + "require_iat": False, + "require_exp": True, + "require_nbf": False, + "require_iss": True, + "require_sub": True, + "require_jti": False, + "require_at_hash": False, + "leeway": 0, + } + data = jwt.decode( + apikey, + self.key, + algorithms=["RS256"], + # TODO verify more besides exp and public key + # like aud, iss, nbf + options=options, + # TODO check issuer is allowed with settings.issuer_whitelist + ) + return User( + username=data["sub"], + roles=data["roles"], + apikey=apikey, + # TODO store issuer in db so we can see from where job was submitted? + ) + + @classmethod + def from_file(cls, public_key: Path) -> "JwtDecoder": + """Create a JwtDecoder from a public key file. + + Args: + public_key: Path to the public key file. + + Returns: + A JwtDecoder object. + """ + public_key_body = public_key.read_bytes() + return cls.from_bytes(public_key_body) + + @classmethod + def from_bytes(cls, public_key: bytes) -> "JwtDecoder": + """Create a JwtDecoder from a public key. + + Args: + public_key: The public key. + + Returns: + A JwtDecoder object. + """ + return cls(jwk.construct(public_key, "RS256")) + + +def generate_token_subcommand( # noqa: WPS211 -- too many arguments + private_key: Path, + username: str, + roles: list[str], + lifetime: int, + issuer: str, + oformat: Literal["header", "plain"] = "plain", +) -> None: + """Generate a JSON Web Token (JWT) with the given parameters. + + Args: + private_key: Path to the private key file. + username: The username to include in the token. + roles: A list of roles to include in the token. + lifetime: The lifetime of the token in minutes. + issuer: The issuer of the token. + oformat: The format of the token output. Can be "header" or "plain". Returns: - User: A User object representing the decoded token. + None """ - options = { - "verify_signature": True, - "verify_aud": True, - "verify_iat": True, - "verify_exp": True, - "verify_nbf": True, - "verify_iss": True, - "verify_sub": True, - "verify_jti": True, - "verify_at_hash": True, - "require_aud": False, - "require_iat": False, - "require_exp": True, - "require_nbf": False, - "require_iss": True, - "require_sub": True, - "require_jti": False, - "require_at_hash": False, - "leeway": 0, - } - data = jwt.decode( - apikey, - public_key, - algorithms=["RS256"], - # TODO verify more besides exp and public key - # like aud, iss, nbf - options=options, - # TODO check issuer is allowed with settings.issuer_whitelist - ) - return User( - username=data["sub"], - roles=data["roles"], - apikey=apikey, - # TODO store issuer in db so we can see from where job was submitted? + private_key_body = Path(private_key).read_bytes() + expire = datetime.utcnow() + timedelta(minutes=lifetime) + token = generate_token( + private_key=private_key_body, + username=username, + roles=roles, + expire=expire, + issuer=issuer, ) + if oformat == "header": + print(f"Authorization: Bearer {token}") # noqa: WPS421 -- user feedback + else: + print(token) # noqa: WPS421 -- user feedback + + +def generate_token( + private_key: bytes, + username: str, + roles: list[str], + expire: datetime, + issuer: str, +) -> str: + """Generate a JSON Web Token (JWT). + + Args: + private_key: The private key to use for signing the token. + username: The username to include in the token. + roles: A list of roles to include in the token. + expire: When token expires. + issuer: The issuer of the token. + + Returns: + The generated token. + """ + # TODO use scope to allow different actions + # no scope could only be used to list applications and check health + # scope:read could be used to read your own job + # scope:write could be used to allow submission/deletion jobs + + # TODO allow super user to read jobs from all users + # by allowing super user to impersonate other users + # with act claim + # see https://www.rfc-editor.org/rfc/rfc8693.html#name-act-actor-claim + # https://auth0.com/docs/secure/tokens/json-web-tokens/json-web-token-claims + # https://www.iana.org/assignments/jwt/jwt.xhtml#claims + # alternativly a super user could also have 'super' role in roles claims + + # TODO allow job to be readable by users who is member of a group + # use groups claims see https://www.iana.org/assignments/jwt/jwt.xhtml#claims + # add group column job table, so on submission we store which group can read + # the job. Add endpoints to add/remove group to/from existing job + + # TODO allow job to be readable by anonymous users aka without token + # Used for storing example jobs or scenarios + # User should have super role. + # Add public boolena column to job table + # Add endpoints to make job public or private + # Public job should not expire + payload = { + "sub": username, + "exp": expire, + "roles": roles, + "iss": issuer, + } + return jwt.encode(payload, private_key, algorithm="RS256") diff --git a/src/bartender/web/lifespan.py b/src/bartender/web/lifespan.py index 0d14097..9d73ac5 100644 --- a/src/bartender/web/lifespan.py +++ b/src/bartender/web/lifespan.py @@ -1,3 +1,4 @@ +import logging from contextlib import asynccontextmanager from typing import AsyncGenerator @@ -11,6 +12,9 @@ teardown_file_staging_queue, ) from bartender.settings import settings +from bartender.user import JwtDecoder + +logger = logging.getLogger(__name__) @asynccontextmanager @@ -26,6 +30,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: _setup_db(app) _parse_context(app) setup_file_staging_queue(app) + setup_jwt_decoder(app) yield await app.state.db_engine.dispose() await close_context(app.state.context) @@ -56,3 +61,18 @@ def _parse_context(app: FastAPI) -> None: config = build_config(settings.config_filename) app.state.config = config app.state.context = build_context(config) + + +def setup_jwt_decoder(app: FastAPI) -> None: + """Setup JWT token decoder. + + Args: + app: fastAPI application. + """ + # TODO read public key from JWKS endpoint from web application that generates tokens + # with settings.jwks = "https://example.com/.well-known/jwks.json" + if settings.public_key.exists(): + app.state.jwt_decoder = JwtDecoder.from_file(settings.public_key) + else: + logger.warning("JWT public key not found, authentication will not work") + app.state.jwt_decoder = None diff --git a/src/bartender/web/users.py b/src/bartender/web/users.py index a7e359e..94d949b 100644 --- a/src/bartender/web/users.py +++ b/src/bartender/web/users.py @@ -1,6 +1,6 @@ from typing import Annotated, Optional -from fastapi import Depends, HTTPException +from fastapi import Depends, HTTPException, Request from fastapi.security import ( APIKeyCookie, APIKeyQuery, @@ -8,10 +8,13 @@ HTTPBearer, ) from jose.exceptions import ExpiredSignatureError, JWTClaimsError, JWTError -from starlette.status import HTTP_401_UNAUTHORIZED, HTTP_403_FORBIDDEN +from starlette.status import ( + HTTP_401_UNAUTHORIZED, + HTTP_403_FORBIDDEN, + HTTP_500_INTERNAL_SERVER_ERROR, +) -from bartender.settings import settings -from bartender.user import User, token2user +from bartender.user import JwtDecoder, User header = HTTPBearer(bearerFormat="jwt", auto_error=False) cookie = APIKeyCookie(name="bartenderToken", auto_error=False) @@ -47,23 +50,46 @@ def current_api_token( raise HTTPException(status_code=HTTP_403_FORBIDDEN, detail="Not authenticated") +def get_jwt_decoder(request: Request) -> JwtDecoder: + """Get JWT decoder from app state. + + Args: + request: current request. + + Raises: + HTTPException: Internal server error 500 if JWT decoder is not setup. + + Returns: + JWT decoder object + """ + try: + return request.app.state.jwt_decoder + except AttributeError: + raise HTTPException( + status_code=HTTP_500_INTERNAL_SERVER_ERROR, + detail="JWT decoder not setup", + ) + + def current_user( apikey: Annotated[str, Depends(current_api_token)], + jwt_decoder: Annotated[JwtDecoder, Depends(get_jwt_decoder)], ) -> User: """Current user based on API token. Args: apikey: API key + jwt_decoder: JWT decoder Raises: HTTPException: Unauthorized 401 response if API key is invalid. + Or internal server error 500 if JWT decoder is not setup. Returns: User """ - public_key = settings.jwt_key try: - return token2user(apikey, public_key) + return jwt_decoder(apikey) except ExpiredSignatureError as exception: raise HTTPException(status_code=HTTP_401_UNAUTHORIZED, detail=str(exception)) except JWTClaimsError as exception: diff --git a/tests/conftest.py b/tests/conftest.py index 5fc0401..4b6968b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,28 +1,23 @@ -import contextlib +from datetime import datetime, timedelta from pathlib import Path -from typing import Any, AsyncGenerator, Callable, Dict, Generator, TypedDict, cast -from uuid import UUID +from typing import Any, AsyncGenerator, Callable, Dict, Generator import pytest from fastapi import FastAPI from httpx import AsyncClient -from sqlalchemy import select +from rsa.key import PrivateKey, PublicKey from sqlalchemy.ext.asyncio import ( AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine, ) -from sqlalchemy.orm import Mapped -from starlette import status from testcontainers.redis import RedisContainer from bartender.config import ApplicatonConfiguration, Config, get_config from bartender.context import Context, get_context from bartender.db.base import Base -from bartender.db.dao.user_dao import get_user_db from bartender.db.dependencies import get_db_session -from bartender.db.models.user import User from bartender.db.utils import create_database, drop_database from bartender.destinations import Destination, DestinationConfig from bartender.filesystems.local import LocalFileSystem, LocalFileSystemConfig @@ -35,7 +30,9 @@ from bartender.picker import pick_first from bartender.schedulers.memory import MemoryScheduler, MemorySchedulerConfig from bartender.settings import settings +from bartender.user import JwtDecoder, User, generate_token from bartender.web.application import get_app +from bartender.web.users import get_jwt_decoder @pytest.fixture(scope="session") @@ -192,12 +189,38 @@ async def demo_file_staging_queue( await stop_file_staging_queue(task) +@pytest.fixture +def rsa_private_key() -> bytes: + # Generating key pair takes a long time, so we use a pre-generated one. + return PrivateKey( + 16527233807183266269587235774365684740774442848897320223621644242162708075059004841994812461818551827813820028470541786594824951798924962833196937230332653347198242638972401822735075360903329143434122437254590842505520000637777584258863217840733503759046884365443875217172374074672632511614003376644212446501060173045618568392384402659352797171568914979982599935263980121157653531287731867867326624158720037621597557065975115568717924320256636245212365326666346692788428023844882564782417147188015148534865863136348561301497209141595301187823693513544644608177477609585687544044700889309819613155368700909410083294879, # noqa: E501 + 65537, + 16036992530939857666384806820562994792560983018599070524753516674425944041033725909287518636562966971118059372118454671939892017635061647030707735009056630976522847309766573830251486144100666954825612221376187427673734430940662372641010247831694549713124929695464735289769790874325292877471773224196003464927888430829809896707363739035492071190604807795076177158622133953913251698586467753318006753650104814965278886101243804987384744165994550506467931731657793604361723532290798162824052964643382452657694969218202591179563541368032631796737297278407443583396564933401588994826686823168536008646497707732114719774753, # noqa: E501 + 3171967619107811069817361866821354854560605851348765969643669560644751648434285601101280062817063639695049513708445583547887807170050138218912545320896949050175144991491657826856804357927098073351793115431246799523325080335397219305826816519226967228208697717903201527652952239459621368951904879478366851746631123668687306758641, # noqa: E501 + 5210404326836076382067892473139554034909041503501665437923474045215820466047899918700828184145139200036788456268704594084460724877064607773425827543403046979191340362536974491902533344983810692931792294710343456528614405630142620960411772083200797551925562926976727904019758115353594552719, # noqa: E501 + ).save_pkcs1() + + +@pytest.fixture +def rsa_publc_key() -> bytes: + return PublicKey( + 16527233807183266269587235774365684740774442848897320223621644242162708075059004841994812461818551827813820028470541786594824951798924962833196937230332653347198242638972401822735075360903329143434122437254590842505520000637777584258863217840733503759046884365443875217172374074672632511614003376644212446501060173045618568392384402659352797171568914979982599935263980121157653531287731867867326624158720037621597557065975115568717924320256636245212365326666346692788428023844882564782417147188015148534865863136348561301497209141595301187823693513544644608177477609585687544044700889309819613155368700909410083294879, # noqa: E501 + 65537, + ).save_pkcs1() + + +@pytest.fixture +def demo_jwt_decoder(rsa_publc_key: bytes) -> JwtDecoder: + return JwtDecoder.from_bytes(rsa_publc_key) + + @pytest.fixture async def fastapi_app( dbsession: AsyncSession, demo_config: Config, demo_context: Context, demo_file_staging_queue: FileStagingQueue, + demo_jwt_decoder: JwtDecoder, ) -> FastAPI: """Fixture for creating FastAPI app. @@ -211,7 +234,7 @@ async def fastapi_app( application.dependency_overrides[ get_file_staging_queue ] = lambda: demo_file_staging_queue - settings.secret = "testsecret" # noqa: S105 + application.dependency_overrides[get_jwt_decoder] = lambda: demo_jwt_decoder return application @@ -232,70 +255,26 @@ async def client( yield ac -async def new_user( - email: str, - password: str, - fastapi_app: FastAPI, - client: AsyncClient, -) -> Any: - new_user = {"email": email, "password": password} - register_url = fastapi_app.url_path_for("register:register") - return await client.post(register_url, json=new_user) - - -MockUser = TypedDict("MockUser", {"id": str, "email": str, "password": str}) +def generate_test_token(rsa_private_key: bytes, username: str, roles: list[str]) -> str: + # Expire long enough in the future so it does not expire during tests. + expire = datetime.utcnow() + timedelta(days=1) + return generate_token( + private_key=rsa_private_key, + username=username, + roles=roles, + issuer="pytest", + expire=expire, + ) @pytest.fixture -async def current_user_model(fastapi_app: FastAPI, client: AsyncClient) -> MockUser: - email = "me@example.com" - password = "mysupersecretpassword" # noqa: S105 needed for tests - response = await new_user(email, password, fastapi_app, client) - body = response.json() - body["password"] = password - return body +def current_user_token(rsa_private_key: bytes) -> str: + return generate_test_token(rsa_private_key, "me@example.com", ["role1"]) @pytest.fixture -def current_user_id(current_user_model: MockUser) -> str: - return current_user_model["id"] - - -async def new_user_token( - email: str, - password: str, - fastapi_app: FastAPI, - client: AsyncClient, -) -> str: - await new_user(email, password, fastapi_app, client) - login_url = fastapi_app.url_path_for("auth:local.login") - login_response = await client.post( - login_url, - data={ - "grant_type": "password", - "username": email, - "password": password, - }, - ) - return login_response.json()["access_token"] - - -@pytest.fixture -async def current_user_token( - fastapi_app: FastAPI, - client: AsyncClient, - current_user_model: MockUser, -) -> str: - """Registers dummy user and returns its auth token. - - :return: token - """ - return await new_user_token( - current_user_model["email"], - current_user_model["password"], - fastapi_app, - client, - ) +async def second_user_token(rsa_private_key: bytes) -> str: + return generate_test_token(rsa_private_key, "user@example.com", []) @pytest.fixture @@ -309,26 +288,11 @@ def auth_headers(current_user_token: str) -> Dict[str, str]: @pytest.fixture -async def current_user(dbsession: AsyncSession, current_user_token: str) -> User: - # User.email is typed as str in fastapi-user package, which confuses mypy, - # cast it to correct type - user_column = cast(Mapped[str], User.email) - query = select(User).where(user_column == "me@example.com") - result = await dbsession.execute(query) - return result.unique().scalar_one() - - -@pytest.fixture -async def second_user_token(fastapi_app: FastAPI, client: AsyncClient) -> str: - """Registers second dummy user and returns its auth token. - - :return: token - """ - return await new_user_token( - "user2@example.com", - "mysupersecretpassword2", - fastapi_app, - client, +def current_user(current_user_token: str) -> User: + return User( + username="me@example.com", + roles=["role1"], + apikey=current_user_token, ) @@ -345,18 +309,6 @@ def redis_dsn(redis_server: RedisContainer) -> str: return f"redis://{host}:{port}/0" -@pytest.fixture -async def current_user_is_super(dbsession: AsyncSession, current_user_id: str) -> None: - # First user can not become super user by calling routes, - # Must make user user super by talking to db directly. - get_user_db_context = contextlib.asynccontextmanager(get_user_db) - async with get_user_db_context(dbsession) as user_db: - user = await user_db.get(UUID(current_user_id)) - if user is None: - raise ValueError(f"User with {current_user_id} id not found") - await user_db.give_super_powers(user) - - @pytest.fixture def app_with_roles( fastapi_app: FastAPI, @@ -364,21 +316,3 @@ def app_with_roles( ) -> FastAPI: demo_applications["app1"].allowed_roles = ["role1"] return fastapi_app - - -@pytest.fixture -async def current_user_with_role( - app_with_roles: FastAPI, - current_user_is_super: None, - current_user_id: str, - auth_headers: Dict[str, str], - client: AsyncClient, -) -> None: - url = app_with_roles.url_path_for( - "assign_role_to_user", - role_id="role1", - user_id=current_user_id, - ) - response = await client.put(url, headers=auth_headers) - assert response.status_code == status.HTTP_200_OK - assert response.json() == ["role1"] diff --git a/tests/test_user.py b/tests/test_user.py new file mode 100644 index 0000000..1ba68bd --- /dev/null +++ b/tests/test_user.py @@ -0,0 +1,31 @@ +from pathlib import Path + +from pytest import CaptureFixture + +from bartender.user import JwtDecoder, generate_token_subcommand + + +def test_generate_token_subcommand( + rsa_private_key: bytes, + demo_jwt_decoder: JwtDecoder, + tmp_path: Path, + capsys: CaptureFixture[str], +) -> None: + private_key_file = tmp_path / "private_key.pem" + private_key_file.write_bytes(rsa_private_key) + + generate_token_subcommand( + private_key=private_key_file, + username="test", + roles=["test"], + lifetime=100, + issuer="test", + oformat="plain", + ) + + captured = capsys.readouterr() + token = captured.out.strip() + user = demo_jwt_decoder(token) + assert user.username == "test" + assert user.roles == ["test"] + assert user.apikey == token diff --git a/tests/web/test_application.py b/tests/web/test_application.py index c47a265..7cf30c7 100644 --- a/tests/web/test_application.py +++ b/tests/web/test_application.py @@ -4,7 +4,6 @@ from typing import Any, Dict, Generator from zipfile import ZipFile -import jwt import pytest from fastapi import FastAPI from httpx import AsyncClient @@ -48,6 +47,7 @@ async def test_upload( job_root_dir: Path, tmp_path: Path, auth_headers: Dict[str, str], + current_user_token: str, ) -> None: """Test upload of a job archive.""" url = fastapi_app.url_path_for("upload_job", application="app1") @@ -61,12 +61,11 @@ async def test_upload( assert job["state"] == "ok" - assert_job_dir(job_root_dir, str(job["id"])) + assert_job_dir(job_root_dir, str(job["id"]), current_user_token) @pytest.mark.anyio async def test_upload_with_role_granted( - current_user_with_role: None, fastapi_app: FastAPI, client: AsyncClient, job_root_dir: Path, @@ -86,21 +85,26 @@ async def test_upload_with_no_role_granted( client: AsyncClient, job_root_dir: Path, tmp_path: Path, - auth_headers: Dict[str, str], + second_user_token: str, ) -> None: url = app_with_roles.url_path_for("upload_job", application="app1") + headers = {"Authorization": f"Bearer {second_user_token}"} with prepare_form_data(tmp_path) as files: - response = await client.put(url, files=files, headers=auth_headers) + response = await client.put(url, files=files, headers=headers) assert response.status_code == status.HTTP_403_FORBIDDEN assert "Missing role" in response.text -def assert_job_dir(job_root_dir: Path, job_id: str) -> None: # noqa: WPS218 +def assert_job_dir( # noqa: WPS218 + job_root_dir: Path, + job_id: str, + current_user_token: str, +) -> None: job_dir = job_root_dir / job_id meta_job_id, meta_job_token = (job_dir / "meta").read_text().splitlines() assert meta_job_id == job_id - assert jwt_decode(meta_job_token) + assert meta_job_token == current_user_token assert (job_dir / "job.ini").read_text() == "# Example config file" assert (job_dir / "input.csv").read_text() == "# Example input data file" assert (job_dir / "stdout.txt").read_text() == " 0 4 21 job.ini\n" @@ -108,15 +112,6 @@ def assert_job_dir(job_root_dir: Path, job_id: str) -> None: # noqa: WPS218 assert (job_dir / "returncode").read_text() == "0" -def jwt_decode(token: str) -> Dict[str, Any]: - # value from fastapi_app fixture - key = "testsecret" - # values from fastapi_users.authentication.JWTStrategy - audience = ["fastapi-users:auth"] - algorithms = ["HS256"] - return jwt.decode(token, key, audience=audience, algorithms=algorithms) - - async def wait_for_job_completion( client: AsyncClient, jurl: str, diff --git a/tests/web/test_job.py b/tests/web/test_job.py index 600ddfb..5aa1868 100644 --- a/tests/web/test_job.py +++ b/tests/web/test_job.py @@ -15,11 +15,11 @@ from bartender.context import Context from bartender.db.dao.job_dao import JobDAO from bartender.db.models.job_model import State -from bartender.db.models.user import User from bartender.destinations import Destination from bartender.filesystems.abstract import AbstractFileSystem from bartender.filesystems.queue import FileStagingQueue from bartender.schedulers.abstract import AbstractScheduler, JobDescription +from bartender.user import User from bartender.web.api.job.views import retrieve_job, retrieve_jobs somedt = datetime(2022, 1, 1, tzinfo=timezone.utc) @@ -36,7 +36,7 @@ async def mock_db_of_job( return await dao.create_job( name="testjob1", application="app1", - submitter=current_user, + submitter=current_user.username, created_on=somedt, updated_on=somedt, ) @@ -398,7 +398,7 @@ async def prepare_job( job_id = await dao.create_job( name="testjob1", application="app1", - submitter=current_user, + submitter=current_user.username, created_on=somedt, updated_on=somedt, ) diff --git a/tests/web/test_role.py b/tests/web/test_role.py deleted file mode 100644 index 59f1a5a..0000000 --- a/tests/web/test_role.py +++ /dev/null @@ -1,144 +0,0 @@ -from typing import Dict - -import pytest -from fastapi import FastAPI -from httpx import AsyncClient -from starlette import status - - -@pytest.mark.anyio -async def test_list_roles( - client: AsyncClient, - app_with_roles: FastAPI, - current_user_is_super: None, - auth_headers: Dict[str, str], -) -> None: - url = app_with_roles.url_path_for( - "list_roles", - ) - response = await client.get(url, headers=auth_headers) - - assert response.status_code == status.HTTP_200_OK - - expected = ["role1"] - assert response.json() == expected - - -@pytest.mark.anyio -async def test_assign_role_to_user( - client: AsyncClient, - fastapi_app: FastAPI, - auth_headers: Dict[str, str], - current_user_with_role: None, -) -> None: - # assign_role_to_user is exercised in current_user_with_role fixture. - - url = fastapi_app.url_path_for( - "profile", - ) - response = await client.get(url, headers=auth_headers) - assert response.status_code == status.HTTP_200_OK - expected = ["role1"] - assert response.json()["roles"] == expected - - -@pytest.mark.anyio -async def test_assign_role_to_user_given_bad_user( - client: AsyncClient, - fastapi_app: FastAPI, - auth_headers: Dict[str, str], - current_user_is_super: None, -) -> None: - # uuid taken from https://en.wikipedia.org/wiki/Universally_unique_identifier - bad_user_id = "123e4567-e89b-12d3-a456-426614174000" - url = fastapi_app.url_path_for( - "assign_role_to_user", - role_id="role1", - user_id=bad_user_id, - ) - response = await client.put(url, headers=auth_headers) - - assert response.status_code == status.HTTP_404_NOT_FOUND - assert "User not found" in response.text - - -@pytest.mark.anyio -async def test_assign_role_to_user_given_bad_role( - client: AsyncClient, - fastapi_app: FastAPI, - auth_headers: Dict[str, str], - current_user_is_super: None, - current_user_id: str, -) -> None: - url = fastapi_app.url_path_for( - "assign_role_to_user", - role_id="badrole1", - user_id=current_user_id, - ) - response = await client.put(url, headers=auth_headers) - - assert response.status_code == status.HTTP_404_NOT_FOUND - assert "Role not found" in response.text - - -@pytest.mark.anyio -async def test_unassign_role_from_user_given_role1_has_been_assigned( - client: AsyncClient, - fastapi_app: FastAPI, - auth_headers: Dict[str, str], - current_user_with_role: None, - current_user_id: str, -) -> None: - url = fastapi_app.url_path_for( - "unassign_role_from_user", - role_id="role1", - user_id=current_user_id, - ) - response = await client.delete(url, headers=auth_headers) - - assert response.status_code == status.HTTP_200_OK - assert not len(response.json()) - - url = fastapi_app.url_path_for("profile") - profile_response = await client.get(url, headers=auth_headers) - assert profile_response.status_code == status.HTTP_200_OK - assert not profile_response.json()["roles"] - - -@pytest.mark.anyio -async def test_unassign_role_from_user_given_bad_user( - client: AsyncClient, - fastapi_app: FastAPI, - auth_headers: Dict[str, str], - current_user_is_super: None, -) -> None: - # uuid taken from https://en.wikipedia.org/wiki/Universally_unique_identifier - bad_user_id = "123e4567-e89b-12d3-a456-426614174000" - url = fastapi_app.url_path_for( - "unassign_role_from_user", - role_id="role1", - user_id=bad_user_id, - ) - response = await client.delete(url, headers=auth_headers) - - assert response.status_code == status.HTTP_404_NOT_FOUND - assert "User not found" in response.text - - -@pytest.mark.anyio -async def test_unassign_role_from_user_given_bad_role( - client: AsyncClient, - fastapi_app: FastAPI, - auth_headers: Dict[str, str], - current_user_is_super: None, - current_user_id: str, -) -> None: - url = fastapi_app.url_path_for( - "unassign_role_from_user", - role_id="badrole1", - user_id=current_user_id, - ) - response = await client.delete(url, headers=auth_headers) - - assert response.status_code == status.HTTP_404_NOT_FOUND - assert "Role not found" in response.text diff --git a/tests/web/test_user.py b/tests/web/test_user.py deleted file mode 100644 index 834f830..0000000 --- a/tests/web/test_user.py +++ /dev/null @@ -1,94 +0,0 @@ -from typing import Dict - -import pytest -from fastapi import FastAPI -from httpx import AsyncClient -from starlette import status - -from tests.conftest import MockUser - - -@pytest.mark.anyio -async def test_profile( - client: AsyncClient, - fastapi_app: FastAPI, - auth_headers: Dict[str, str], -) -> None: - """Checks the profile endpoint. - - Args: - client: client for the app. - fastapi_app: current FastAPI application. - """ - url = fastapi_app.url_path_for( - "profile", - ) - - response = await client.get(url, headers=auth_headers) - - assert response.status_code == status.HTTP_200_OK - - expected = { - "email": "me@example.com", - "oauth_accounts": [], - "roles": [], - } - assert response.json() == expected - - -@pytest.mark.anyio -async def test_list_users( - client: AsyncClient, - app_with_roles: FastAPI, - current_user_is_super: None, - current_user_with_role: None, - current_user_model: MockUser, - auth_headers: Dict[str, str], -) -> None: - url = app_with_roles.url_path_for( - "list_users", - ) - response = await client.get(url, headers=auth_headers) - - assert response.status_code == status.HTTP_200_OK - - expected = [ - { - "email": current_user_model["email"], - "id": current_user_model["id"], - "oauth_accounts": [], - "roles": ["role1"], - "is_active": True, - "is_superuser": True, - "is_verified": False, - }, - ] - assert response.json() == expected - - -@pytest.mark.anyio -async def test_list_users_given_current_user_is_not_super( - client: AsyncClient, - app_with_roles: FastAPI, - second_user_token: str, -) -> None: - auth_headers = {"Authorization": f"Bearer {second_user_token}"} - url = app_with_roles.url_path_for( - "list_users", - ) - response = await client.get(url, headers=auth_headers) - - assert response.status_code == status.HTTP_403_FORBIDDEN - - -@pytest.mark.anyio -async def test_list_users_given_anonymous_user( - client: AsyncClient, - app_with_roles: FastAPI, -) -> None: - url = app_with_roles.url_path_for( - "list_users", - ) - response = await client.get(url) - - assert response.status_code == status.HTTP_401_UNAUTHORIZED From d3718df5d739f826baac1a0ded38a08f0bb70410 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Wed, 19 Jul 2023 13:58:03 +0200 Subject: [PATCH 07/10] Add public key in docker compose Fixes #51 --- deploy/Dockerfile | 2 ++ deploy/README.md | 1 + deploy/docker-compose.dirac.yml | 15 ++++++++++----- deploy/docker-compose.yml | 19 +++++++++++++++++-- docs/deploy.md | 21 +++++++++++++++++++++ docs/index.md | 1 + 6 files changed, 52 insertions(+), 7 deletions(-) create mode 120000 deploy/README.md create mode 100644 docs/deploy.md diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 0d55c69..2ab9e3a 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -23,6 +23,8 @@ RUN poetry install --without=dev --no-interaction --no-ansi ENV BARTENDER_HOST=0.0.0.0 ENV BARTENDER_PORT=8000 +# Mounting config.yaml and public_key.pem should be done when running the container + CMD ["/usr/local/bin/bartender", "serve"] HEALTHCHECK --interval=5m --timeout=3s CMD curl -f http://${BARTENDER_HOST}:${BARTENDER_PORT}/api/health || exit 1 diff --git a/deploy/README.md b/deploy/README.md new file mode 120000 index 0000000..89196b5 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1 @@ +../docs/deploy.md \ No newline at end of file diff --git a/deploy/docker-compose.dirac.yml b/deploy/docker-compose.dirac.yml index 5ceded6..f0b9cf0 100644 --- a/deploy/docker-compose.dirac.yml +++ b/deploy/docker-compose.dirac.yml @@ -31,10 +31,17 @@ services: BARTENDER_DB_USER: bartender BARTENDER_DB_PASS: bartender BARTENDER_DB_BASE: bartender + BARTENDER_PUBLIC_KEY: /app/src/public_key.pem volumes: - type: bind source: ../config.yaml target: /app/src/config.yaml + - type: bind + source: ../public_key.pem + target: /app/src/public_key.pem + - type: volume + source: bartender-jobs + target: /tmp/jobs db: image: postgres:15.2-bullseye @@ -55,7 +62,7 @@ services: migrator: image: bartender-with-dirac:${BARTENDER_VERSION:-latest} restart: "no" - command: '"alembic upgrade head"' + command: alembic upgrade head environment: BARTENDER_DB_HOST: bartender-db BARTENDER_DB_PORT: 5432 @@ -65,11 +72,9 @@ services: depends_on: db: condition: service_healthy - volumes: - - type: bind - source: ../config.yaml - target: /app/src/config.yaml volumes: + bartender-jobs: + name: bartender-jobs bartender-db-data: name: bartender-db-data diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index bc11f53..5df4261 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -3,12 +3,14 @@ version: '3.9' services: api: build: - context: . + context: .. dockerfile: ./deploy/Dockerfile image: bartender:${BARTENDER_VERSION:-latest} + ports: + - "8000:8000" restart: always env_file: - - .env + - ../.env depends_on: db: condition: service_healthy @@ -19,6 +21,17 @@ services: BARTENDER_DB_USER: bartender BARTENDER_DB_PASS: bartender BARTENDER_DB_BASE: bartender + BARTENDER_PUBLIC_KEY: /app/src/public_key.pem + volumes: + - type: bind + source: ../config.yaml + target: /app/src/config.yaml + - type: bind + source: ../public_key.pem + target: /app/src/public_key.pem + - type: volume + source: bartender-jobs + target: /tmp/jobs db: image: postgres:15.2-bullseye @@ -51,5 +64,7 @@ services: condition: service_healthy volumes: + bartender-jobs: + name: bartender-jobs bartender-db-data: name: bartender-db-data diff --git a/docs/deploy.md b/docs/deploy.md new file mode 100644 index 0000000..1b9fc9c --- /dev/null +++ b/docs/deploy.md @@ -0,0 +1,21 @@ +# Deploy with docker compose + +Create config file `config.yaml` as described at [configuration.md](configuration.md). +The `job_root_dir` property should be set to `/tmp/jobs` +which is a Docker compose volume. + +Store public RSA key for JWT auth in `public_key.pem` file next to `config.yaml`. + +Start with + +```bash +docker compose -f deploy/docker-compose.yml up +``` + +Web service will running on . + +To login to web service you need to generate token with +the private counterpart of the public key. +See [configuration.md#authentication](configuration.md#authentication). +To use `bartender generate-token` command inside container you need make +the private key available in the container. diff --git a/docs/index.md b/docs/index.md index 012bd72..768cf84 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,4 +9,5 @@ self develop configuration +deploy ``` From 21cb45e907818297f9be149a5c2da6c2a9c5fcf5 Mon Sep 17 00:00:00 2001 From: sverhoeven Date: Fri, 28 Jul 2023 13:18:06 +0200 Subject: [PATCH 08/10] Make role claim in JWT optional --- src/bartender/user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bartender/user.py b/src/bartender/user.py index 3ebddcf..d0aa024 100644 --- a/src/bartender/user.py +++ b/src/bartender/user.py @@ -68,7 +68,7 @@ def __call__(self, apikey: str) -> User: ) return User( username=data["sub"], - roles=data["roles"], + roles=data["roles"] if "roles" in data else [], apikey=apikey, # TODO store issuer in db so we can see from where job was submitted? ) From 1505b9c1157b3db0579c6374a09a81f983287642 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Mon, 21 Aug 2023 13:23:49 +0200 Subject: [PATCH 09/10] Move allowed_roles to admin app + Add bind example for private key --- .dockerignore | 2 ++ deploy/docker-compose.yml | 5 +++++ docs/configuration.md | 14 +++++++++----- docs/deploy.md | 9 +++++---- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/.dockerignore b/.dockerignore index d7a291e..2547251 100644 --- a/.dockerignore +++ b/.dockerignore @@ -145,3 +145,5 @@ cython_debug/ # The app config /config.yaml +/private_key.pem +/public_key.pem diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 5df4261..9507f00 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -29,6 +29,11 @@ services: - type: bind source: ../public_key.pem target: /app/src/public_key.pem + # If you want to generate a token for testing purposes with the `bartender generate-token` command + # also mount private key by uncommenting the following lines + # - type: bind + # source: ../private_key.pem + # target: /app/src/private_key.pem - type: volume source: bartender-jobs target: /tmp/jobs diff --git a/docs/configuration.md b/docs/configuration.md index 5351256..bcadd50 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -42,7 +42,7 @@ openssl genpkey -algorithm RSA -out private_key.pem -pkeyopt rsa_keygen_bits:204 openssl rsa -pubout -in private_key.pem -out public_key.pem ``` -The private key of the RSA key pair is used to generate a token in +The private key of the RSA key pair is used to sign a token in an another web application or with the `bartender generate-token` command. The public key of the RSA key pair is used to verify that the token comes @@ -55,7 +55,7 @@ The token payload should contain the following claims: * `sub`: The user id. Used to identifiy who submitted a job. * `exp`: The expiration time of the token. * `iss`: The issuer of the token. Used to track from where jobs are submitted. -* `roles`: The roles of the user. +* `roles`: Optionally. The roles of the user. See [Applications](#applications) how roles are used. ## Configuration file @@ -87,8 +87,11 @@ applications: haddock3: command: haddock3 $config config: workflow.cfg + adminapp: + command: some-admin-application $config + config: config.yaml allowed_roles: - - easy + - admin # Only users with admin role can submit jobs for this application ``` * The key is the name of the application @@ -97,8 +100,9 @@ applications: * The `command` key is the command executed in the directory of the unpacked archive that the consumer uploaded. The `$config` in command string will be replaced with value of the config key. -* The `allowed_roles` key holds an array of role names, one of which a submitter - should have. When key is not set or list is empty then any authorized user +* Optionally, the `allowed_roles` key holds an array of role names, + one of which a submitter should have. + When key is not set or list is empty then any authorized user is allowed. See [Authentication](#authentication) how to set roles on users. * The application command should not overwrite files uploaded during submission as these might not be downloaded from location where application is run. diff --git a/docs/deploy.md b/docs/deploy.md index 1b9fc9c..051d0bb 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -14,8 +14,9 @@ docker compose -f deploy/docker-compose.yml up Web service will running on . -To login to web service you need to generate token with -the private counterpart of the public key. +To login to web service you need to generate token and sign it with +the private counterpart of the public key.g +If you want to generate a token with the +`docker compose -f deploy/docker-compose.yml exec api bartender generate-token` command +you should uncomment the private key volume bind in `deploy/docker-compose.yml`. See [configuration.md#authentication](configuration.md#authentication). -To use `bartender generate-token` command inside container you need make -the private key available in the container. From a20d0980bf2e5900aaabbb1c0290bd1e2af41637 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Mon, 21 Aug 2023 13:31:11 +0200 Subject: [PATCH 10/10] Move job root dir chapter up --- docs/configuration.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index bcadd50..91a5153 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -70,6 +70,16 @@ is shipped with the repository. Here, we explain the options in more detail. +## Job root dir + +By default, the files of jobs are stored in `/tmp/jobs`. To change the +directory, set the `job_root_dir` parameter in the configuration file to a valid +path. + +```yaml +job_root_dir: /tmp/jobs +``` + ## Applications Bartender accepts jobs for different applications. @@ -367,16 +377,6 @@ destination use: destination_picker: bartender.picker.pick_round ``` -## Job root dir - -By default, the files of jobs are stored in `/tmp/jobs`. To change the -directory, set the `job_root_dir` parameter in the configuration file to a valid -path. - -```yaml -job_root_dir: /tmp/jobs -``` - ## Job flow Diagram of a job flowing through web service, schedulers and filesystems.