25459641641

Committed 06 May 2026 08:35PM UTC coverage: 75.077% (-0.001%) from 75.078%

Build # 25459641641

Build Type

push

github

Committed by

PlanXCyborg

Commit Message

Merge https://github.com/uc-cdis/fence into stable

Coverage Stats

8489 of 11307 relevant lines covered (75.08%)

0.75 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.8

fence/__init__.py

# Override the default_digest_method for Signer before flask and flask_wtf are loaded.
import hashlib
from itsdangerous import Signer

# Explicitly set to sha256 as the default (sha1) will break in FIPS environments when flask_wtf attempts to process a user registration form.
# This is a known issue with itsdangerous defaults (see: https://github.com/pgadmin-org/pgadmin4/issues/7979 for a similar issue with pgadmin)
# According to: https://itsdangerous.palletsprojects.com/en/latest/concepts/#digest-method-security and https://stackoverflow.com/a/27669587, we can override the default here:
Signer.default_digest_method = hashlib.sha256

from collections import OrderedDict
import os
from urllib.parse import urljoin

from authutils.oauth2.client import OAuthClient
from azure.storage.blob import BlobServiceClient
from azure.core.exceptions import ResourceNotFoundError
from cdislogging import get_logger
import flask
from flask_cors import CORS
from flask_wtf.csrf import validate_csrf
from gen3authz.client.arborist.client import ArboristClient
from sqlalchemy.orm import scoped_session


# Can't read config yet. Just set to debug for now, else no handlers.
# Later, in app_config(), will actually set level based on config
logger = get_logger(__name__, log_level="debug")

# Load the configuration *before* importing modules that rely on it
from fence.config import config, CONFIG_SEARCH_FOLDERS

config.load(
    config_path=os.environ.get("FENCE_CONFIG_PATH"),
    search_folders=CONFIG_SEARCH_FOLDERS,
)

from fence.auth import logout, build_redirect_url
from fence.metrics import metrics
from fence.blueprints.data.indexd import S3IndexedFileLocation
from fence.errors import UserError
from fence.jwt import keys
from fence.oidc.client import query_client
from fence.oidc.server import server
from fence.resources.audit.client import AuditServiceClient
from fence.resources.aws.boto_manager import BotoManager
from fence.resources.openid.idp_oauth2 import Oauth2ClientBase
from fence.resources.openid.cilogon_oauth2 import CilogonOauth2Client
from fence.resources.openid.cognito_oauth2 import CognitoOauth2Client
from fence.resources.openid.google_oauth2 import GoogleOauth2Client
from fence.resources.openid.microsoft_oauth2 import MicrosoftOauth2Client
from fence.resources.openid.okta_oauth2 import OktaOauth2Client
from fence.resources.openid.orcid_oauth2 import OrcidOauth2Client
from fence.resources.openid.synapse_oauth2 import SynapseOauth2Client
from fence.resources.openid.ras_oauth2 import RASOauth2Client
from fence.resources.storage import StorageManager
from fence.resources.user.user_session import UserSessionInterface
from fence.error_handler import get_error_response
from fence.utils import get_SQLAlchemyDriver, allowed_login_redirects, domain
import fence.blueprints.admin
import fence.blueprints.data
import fence.blueprints.login
import fence.blueprints.oauth2
import fence.blueprints.misc
import fence.blueprints.storage_creds
import fence.blueprints.user
import fence.blueprints.well_known
import fence.blueprints.link
import fence.blueprints.google
import fence.blueprints.privacy
import fence.blueprints.register
import fence.blueprints.ga4gh


app = flask.Flask(__name__)
CORS(app=app, headers=["content-type", "accept"], expose_headers="*")


def warn_about_logger():
    raise Exception(
        "Flask 0.12 will remove and replace all of our log handlers if you call "
        "app.logger anywhere. Use get_logger from cdislogging instead."
    )


def app_init(
    app,
    root_dir=None,
    config_path=None,
    config_file_name=None,
):
    app.__dict__["logger"] = warn_about_logger

    app_config(
        app,
        root_dir=root_dir,
        config_path=config_path,
        file_name=config_file_name,
    )
    app_sessions(app)
    app_register_blueprints(app)
    server.init_app(app, query_client=query_client)
    logger.info(
        f"Prometheus metrics are{'' if config['ENABLE_PROMETHEUS_METRICS'] else ' NOT'} enabled."
    )


def app_sessions(app):
    app.url_map.strict_slashes = False
    app.db = get_SQLAlchemyDriver(config["DB"])

    # app.db.Session is from SQLAlchemyDriver and uses
    # SQLAlchemy's sessionmaker. Using scoped_session here ensures
    # a thread-local db session is created. Effectively the below is expanded to:
    #   app.scoped_session = scoped_session(
    #       sessionmaker(
    #            bind=sqlalchemy.create_engine(config["DB"]),
    #            expire_on_commit=False,
    #       )
    #   )
    #
    # From the sqlalchemy docs: "The scoped_session object by default uses
    # [Python's threading.local() construct] as
    # storage, so that a single Session is maintained for all who call upon the
    # scoped_session registry, but only within the scope of a single thread.
    # Callers who call upon the registry in a different thread get a Session
    # instance that is local to that other thread."
    app.scoped_session = scoped_session(app.db.Session)

    app.session_interface = UserSessionInterface()


def app_register_blueprints(app):
    app.register_blueprint(fence.blueprints.oauth2.blueprint, url_prefix="/oauth2")
    app.register_blueprint(fence.blueprints.user.blueprint, url_prefix="/user")

    creds_blueprint = fence.blueprints.storage_creds.make_creds_blueprint()
    app.register_blueprint(creds_blueprint, url_prefix="/credentials")

    app.register_blueprint(fence.blueprints.admin.blueprint, url_prefix="/admin")
    app.register_blueprint(
        fence.blueprints.well_known.blueprint, url_prefix="/.well-known"
    )

    login_blueprint = fence.blueprints.login.make_login_blueprint()
    app.register_blueprint(login_blueprint, url_prefix="/login")

    link_blueprint = fence.blueprints.link.make_link_blueprint()
    app.register_blueprint(link_blueprint, url_prefix="/link")

    google_blueprint = fence.blueprints.google.make_google_blueprint()
    app.register_blueprint(google_blueprint, url_prefix="/google")

    app.register_blueprint(
        fence.blueprints.privacy.blueprint, url_prefix="/privacy-policy"
    )

    app.register_blueprint(fence.blueprints.register.blueprint, url_prefix="/register")
    app.register_blueprint(fence.blueprints.ga4gh.blueprint, url_prefix="/ga4gh")

    fence.blueprints.misc.register_misc(app)

    @app.route("/")
    def root():
        """
        Register the root URL.
        """
        endpoints = {
            "oauth2 endpoint": "/oauth2",
            "user endpoint": "/user",
            "keypair endpoint": "/credentials",
        }
        return flask.jsonify(endpoints)

    @app.route("/logout")
    def logout_endpoint():
        root = config.get("BASE_URL", "")
        request_next = flask.request.args.get("next", root)
        force_era_global_logout = (
            flask.request.args.get("force_era_global_logout") == "true"
        )
        if request_next.startswith("https") or request_next.startswith("http"):
            next_url = request_next
        else:
            next_url = build_redirect_url(config.get("ROOT_URL", ""), request_next)
        if domain(next_url) not in allowed_login_redirects():
            raise UserError("invalid logout redirect URL: {}".format(next_url))
        return logout(
            next_url=next_url, force_era_global_logout=force_era_global_logout
        )

    @app.route("/jwt/keys")
    def public_keys():
        """
        Return the public keys which can be used to verify JWTs signed by fence.

        The return value should look like this:
            {
                "keys": [
                    {
                        "key-01": " ... [public key here] ... "
                    }
                ]
            }
        """
        return flask.jsonify(
            {"keys": [(keypair.kid, keypair.public_key) for keypair in app.keypairs]}
        )

    @app.route("/metrics")
    def metrics_endpoint():
        """
        WARNING: There is no authz control on this endpoint!
        In cloud-automation setups, access to this endpoint is blocked at the revproxy level.
        """
        data, content_type = metrics.get_latest_metrics()
        return flask.Response(data, content_type=content_type)


def _check_azure_storage(app):
    """
    Confirm access to Azure Storage Account and Containers
    """
    azure_creds = config.get("AZ_BLOB_CREDENTIALS", None)

    # if this is a public bucket, Fence will not try to sign the URL
    if azure_creds == "*":
        return

    if not azure_creds or azure_creds.strip() == "":
        # Azure Blob credentials are not configured.
        # If you're using Azure Blob Storage set AZ_BLOB_CREDENTIALS to your Azure Blob Storage Connection String.
        logger.debug(
            "Azure Blob credentials are not configured.  If you're using Azure Blob Storage, please set AZ_BLOB_CREDENTIALS to your Azure Blob Storage Connection String."
        )
        return

    blob_service_client = BlobServiceClient.from_connection_string(azure_creds)

    for c in blob_service_client.list_containers():
        container_client = blob_service_client.get_container_client(c.name)

        # check if container exists.  If it doesn't exist, log a warning.
        if container_client.exists() is False:
            logger.debug(
                f"Unable to access Azure Blob Storage Container {c.name}. You may run into issues resolving orphaned indexed files pointing to this container."
            )
            continue

        # verify that you can check the container properties
        try:
            container_properties = container_client.get_container_properties()
            public_access = container_properties["public_access"]
            # check container properties
            logger.debug(
                f"Azure Blob Storage Container {c.name} has public access {public_access}"
            )
        except ResourceNotFoundError as err:
            logger.debug(
                f"Unable to access Azure Blob Storage Container {c.name}. You may run into issues resolving orphaned indexed files pointing to this container."
            )
            logger.debug(err)


def _check_buckets_aws_creds_and_region(app):
    """
    Function to ensure that all s3_buckets have a valid credential.
    Additionally, if there is no region it will produce a warning
    then try to fetch and cache the region.
    """
    buckets = config.get("S3_BUCKETS") or {}
    aws_creds = config.get("AWS_CREDENTIALS") or {}

    # check that AWS creds and regions are configured
    for bucket_name, bucket_details in buckets.items():
        cred = bucket_details.get("cred")
        region = bucket_details.get("region")
        if not cred:
            raise ValueError(
                "No cred for S3_BUCKET: {}. cred is required.".format(bucket_name)
            )

        # if this is a public bucket, Fence will not try to sign the URL
        # so it won't need to know the region.
        if cred == "*":
            continue

        if cred not in aws_creds:
            raise ValueError(
                "Credential {} for S3_BUCKET {} is not defined in AWS_CREDENTIALS".format(
                    cred, bucket_name
                )
            )

        # only require region when we're not specifying an
        # s3-compatible endpoint URL (ex: no need for region when using cleversafe)
        if not region and not bucket_details.get("endpoint_url"):
            logger.warning(
                "WARNING: no region for S3_BUCKET: {}. Providing the region will reduce"
                " response time and avoid a call to GetBucketLocation which you make lack the AWS ACLs for.".format(
                    bucket_name
                )
            )
            credential = S3IndexedFileLocation.get_credential_to_access_bucket(
                bucket_name,
                aws_creds,
                config.get("MAX_PRESIGNED_URL_TTL", 3600),
                app.boto,
            )
            if not getattr(app, "boto"):
                logger.warning(
                    "WARNING: boto not setup for app, probably b/c "
                    "nothing in AWS_CREDENTIALS. Cannot attempt to get bucket "
                    "bucket regions."
                )
                return

            region = app.boto.get_bucket_region(bucket_name, credential)
            config["S3_BUCKETS"][bucket_name]["region"] = region

    cred = config["PUSH_AUDIT_LOGS_CONFIG"].get("aws_sqs_config", {}).get("aws_cred")
    if cred and cred not in aws_creds:
        raise ValueError(
            "Credential {} for PUSH_AUDIT_LOGS_CONFIG.aws_sqs_config.aws_cred is not defined in AWS_CREDENTIALS".format(
                cred
            )
        )

    # check that all the configured buckets are in `S3_BUCKETS`
    bucket_names = config["ALLOWED_DATA_UPLOAD_BUCKETS"] or []
    if config["DATA_UPLOAD_BUCKET"]:
        bucket_names.append(config["DATA_UPLOAD_BUCKET"])
    for bucket_name in bucket_names:
        if bucket_name not in buckets:
            logger.warning(
                f"Data upload bucket '{bucket_name}' is not configured in 'S3_BUCKETS'"
            )


def app_config(
    app,
    root_dir=None,
    config_path=None,
    file_name=None,
):
    """
    Set up the config for the Flask app.
    """
    if root_dir is None:
        root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))

    # logger.info("Loading settings...")
    settings_cfg = flask.Config(app.config.root_path)

    # # dump the settings into the config singleton before loading a configuration file
    config.update(dict(settings_cfg))

    # load the configuration file
    config.load(
        config_path=config_path,
        search_folders=CONFIG_SEARCH_FOLDERS,
        file_name=file_name,
    )

    # load all config back into flask app config for now, we should PREFER getting config
    # directly from the fence config singleton in the code though.
    app.config.update(**config._configs)

    _setup_arborist_client(app)
    _setup_audit_service_client(app)
    _setup_data_endpoint_and_boto(app)
    _load_keys(app, root_dir)

    app.storage_manager = StorageManager(config["STORAGE_CREDENTIALS"], logger=logger)

    app.debug = config["DEBUG"]
    # Following will update logger level, propagate, and handlers
    get_logger(__name__, log_level="debug" if config["DEBUG"] is True else "info")

    _setup_oidc_clients(app)

    with app.app_context():
        _check_buckets_aws_creds_and_region(app)
        _check_azure_storage(app)


def _setup_data_endpoint_and_boto(app):
    if "AWS_CREDENTIALS" in config and len(config["AWS_CREDENTIALS"]) > 0:
        creds = config["AWS_CREDENTIALS"]
        buckets = config.get("S3_BUCKETS", {})
        app.boto = BotoManager(creds, buckets, logger=logger)
        app.register_blueprint(fence.blueprints.data.blueprint, url_prefix="/data")


def _load_keys(app, root_dir):
    if root_dir is None:
        root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))

    app.keypairs = keys.load_keypairs(os.path.join(root_dir, "keys"))

    app.jwt_public_keys = {
        config["BASE_URL"]: OrderedDict(
            [(str(keypair.kid), str(keypair.public_key)) for keypair in app.keypairs]
        )
    }


def _setup_oidc_clients(app):
    configured_idps = config.get("OPENID_CONNECT", {})

    clean_idps = [idp.lower().replace(" ", "") for idp in configured_idps]
    if len(clean_idps) != len(set(clean_idps)):
        raise ValueError(
            f"Some IDPs configured in OPENID_CONNECT are not unique once they are lowercased and spaces are removed: {clean_idps}"
        )

    for idp in set(configured_idps.keys()):
        logger.info(f"Setting up OIDC client for {idp}")
        settings = configured_idps[idp]
        if idp == "google":
            app.google_client = GoogleOauth2Client(
                settings,
                HTTP_PROXY=config.get("HTTP_PROXY"),
                logger=logger,
            )
        elif idp == "orcid":
            app.orcid_client = OrcidOauth2Client(
                settings,
                HTTP_PROXY=config.get("HTTP_PROXY"),
                logger=logger,
            )
        elif idp == "ras":
            app.ras_client = RASOauth2Client(
                settings,
                HTTP_PROXY=config.get("HTTP_PROXY"),
                logger=logger,
            )
        elif idp == "synapse":
            app.synapse_client = SynapseOauth2Client(
                settings, HTTP_PROXY=config.get("HTTP_PROXY"), logger=logger
            )
        elif idp == "microsoft":
            app.microsoft_client = MicrosoftOauth2Client(
                settings,
                HTTP_PROXY=config.get("HTTP_PROXY"),
                logger=logger,
            )
        elif idp == "okta":
            app.okta_client = OktaOauth2Client(
                settings,
                HTTP_PROXY=config.get("HTTP_PROXY"),
                logger=logger,
            )
        elif idp == "cognito":
            app.cognito_client = CognitoOauth2Client(
                settings, HTTP_PROXY=config.get("HTTP_PROXY"), logger=logger
            )
        elif idp == "cilogon":
            app.cilogon_client = CilogonOauth2Client(
                settings,
                HTTP_PROXY=config.get("HTTP_PROXY"),
                logger=logger,
            )
        elif idp == "fence":
            # https://docs.authlib.org/en/latest/client/flask.html#flask-client
            app.fence_client = OAuthClient(app)
            # https://docs.authlib.org/en/latest/client/frameworks.html
            app.fence_client.register(**settings)
        else:  # generic OIDC implementation
            if hasattr(app, "arborist"):
                app_arborist = app.arborist
            else:
                app_arborist = None
            client = Oauth2ClientBase(
                settings=settings,
                logger=logger,
                HTTP_PROXY=config.get("HTTP_PROXY"),
                idp=settings.get("name") or idp.title(),
                arborist=app_arborist,
            )
            clean_idp = idp.lower().replace(" ", "")
            setattr(app, f"{clean_idp}_client", client)


def _setup_arborist_client(app):
    if app.config.get("ARBORIST"):
        app.arborist = ArboristClient(
            arborist_base_url=config["ARBORIST"],
            timeout=app.config.get("ARBORIST_TIMEOUT", 30),
        )
    else:
        logger.info("Arborist not configured")
        app.arborist = None


def _setup_audit_service_client(app):
    # Initialize the client regardless of whether audit logs are enabled. This
    # allows us to call `app.audit_service_client.create_x_log()` from
    # anywhere without checking if audit logs are enabled. The client
    # checks that for us.
    service_url = app.config.get("AUDIT_SERVICE") or urljoin(
        app.config["BASE_URL"], "/audit"
    )
    app.audit_service_client = AuditServiceClient(
        service_url=service_url, logger=logger
    )


@app.errorhandler(Exception)
def handle_error(error):
    """
    Register an error handler for general exceptions.
    """
    return get_error_response(error)


@app.before_request
def check_csrf():
    has_auth = "Authorization" in flask.request.headers
    no_username = not flask.session.get("username")
    if has_auth or no_username:
        return
    if not config.get("ENABLE_CSRF_PROTECTION", True):
        return
    if flask.request.method != "GET":
        try:
            csrf_header = flask.request.headers.get("x-csrf-token")
            csrf_formfield = flask.request.form.get("csrf_token")
            # validate_csrf checks the input (a signed token) against the raw
            # token stored in session["csrf_token"].
            # (session["csrf_token"] is managed by flask-wtf.)
            # To pass CSRF check, there must exist EITHER an x-csrf-token header
            # OR a csrf_token form field that matches the token in the session.
            assert (
                csrf_header
                and validate_csrf(csrf_header) is None
                or csrf_formfield
                and validate_csrf(csrf_formfield) is None
            )

            referer = flask.request.headers.get("referer")
            assert referer, "Referer header missing"
            logger.debug("HTTP REFERER " + str(referer))
        except Exception as e:
            raise UserError("CSRF verification failed: {}. Request aborted".format(e))


@app.teardown_appcontext
def remove_scoped_session(*args, **kwargs):
    if hasattr(app, "scoped_session"):
        try:
            app.scoped_session.remove()
        except Exception as exc:
            logger.warning(f"could not remove app.scoped_session. Error: {exc}")

1	# Override the default_digest_method for Signer before flask and flask_wtf are loaded.
2	import hashlib	1✔
3	from itsdangerous import Signer	1✔
4
5	# Explicitly set to sha256 as the default (sha1) will break in FIPS environments when flask_wtf attempts to process a user registration form.
6	# This is a known issue with itsdangerous defaults (see: https://github.com/pgadmin-org/pgadmin4/issues/7979 for a similar issue with pgadmin)
7	# According to: https://itsdangerous.palletsprojects.com/en/latest/concepts/#digest-method-security and https://stackoverflow.com/a/27669587, we can override the default here:
8	Signer.default_digest_method = hashlib.sha256	1✔
9
10	from collections import OrderedDict	1✔
11	import os	1✔
12	from urllib.parse import urljoin	1✔
13
14	from authutils.oauth2.client import OAuthClient	1✔
15	from azure.storage.blob import BlobServiceClient	1✔
16	from azure.core.exceptions import ResourceNotFoundError	1✔
17	from cdislogging import get_logger	1✔
18	import flask	1✔
19	from flask_cors import CORS	1✔
20	from flask_wtf.csrf import validate_csrf	1✔
21	from gen3authz.client.arborist.client import ArboristClient	1✔
22	from sqlalchemy.orm import scoped_session	1✔
23
24
25	# Can't read config yet. Just set to debug for now, else no handlers.
26	# Later, in app_config(), will actually set level based on config
27	logger = get_logger(__name__, log_level="debug")	1✔
28
29	# Load the configuration before importing modules that rely on it
30	from fence.config import config, CONFIG_SEARCH_FOLDERS	1✔
31
32	config.load(	1✔
33	config_path=os.environ.get("FENCE_CONFIG_PATH"),
34	search_folders=CONFIG_SEARCH_FOLDERS,
35	)
36
37	from fence.auth import logout, build_redirect_url	1✔
38	from fence.metrics import metrics	1✔
39	from fence.blueprints.data.indexd import S3IndexedFileLocation	1✔
40	from fence.errors import UserError	1✔
41	from fence.jwt import keys	1✔
42	from fence.oidc.client import query_client	1✔
43	from fence.oidc.server import server	1✔
44	from fence.resources.audit.client import AuditServiceClient	1✔
45	from fence.resources.aws.boto_manager import BotoManager	1✔
46	from fence.resources.openid.idp_oauth2 import Oauth2ClientBase	1✔
47	from fence.resources.openid.cilogon_oauth2 import CilogonOauth2Client	1✔
48	from fence.resources.openid.cognito_oauth2 import CognitoOauth2Client	1✔
49	from fence.resources.openid.google_oauth2 import GoogleOauth2Client	1✔
50	from fence.resources.openid.microsoft_oauth2 import MicrosoftOauth2Client	1✔
51	from fence.resources.openid.okta_oauth2 import OktaOauth2Client	1✔
52	from fence.resources.openid.orcid_oauth2 import OrcidOauth2Client	1✔
53	from fence.resources.openid.synapse_oauth2 import SynapseOauth2Client	1✔
54	from fence.resources.openid.ras_oauth2 import RASOauth2Client	1✔
55	from fence.resources.storage import StorageManager	1✔
56	from fence.resources.user.user_session import UserSessionInterface	1✔
57	from fence.error_handler import get_error_response	1✔
58	from fence.utils import get_SQLAlchemyDriver, allowed_login_redirects, domain	1✔
59	import fence.blueprints.admin	1✔
60	import fence.blueprints.data	1✔
61	import fence.blueprints.login	1✔
62	import fence.blueprints.oauth2	1✔
63	import fence.blueprints.misc	1✔
64	import fence.blueprints.storage_creds	1✔
65	import fence.blueprints.user	1✔
66	import fence.blueprints.well_known	1✔
67	import fence.blueprints.link	1✔
68	import fence.blueprints.google	1✔
69	import fence.blueprints.privacy	1✔
70	import fence.blueprints.register	1✔
71	import fence.blueprints.ga4gh	1✔
72
73
74	app = flask.Flask(__name__)	1✔
75	CORS(app=app, headers=["content-type", "accept"], expose_headers="*")	1✔
76
77
78	def warn_about_logger():	1✔
79	raise Exception(	×
80	"Flask 0.12 will remove and replace all of our log handlers if you call "
81	"app.logger anywhere. Use get_logger from cdislogging instead."
82	)
83
84
85	def app_init(	1✔
86	app,
87	root_dir=None,
88	config_path=None,
89	config_file_name=None,
90	):
91	app.__dict__["logger"] = warn_about_logger	1✔
92
93	app_config(	1✔
94	app,
95	root_dir=root_dir,
96	config_path=config_path,
97	file_name=config_file_name,
98	)
99	app_sessions(app)	1✔
100	app_register_blueprints(app)	1✔
101	server.init_app(app, query_client=query_client)	1✔
102	logger.info(	1✔
103	f"Prometheus metrics are{'' if config['ENABLE_PROMETHEUS_METRICS'] else ' NOT'} enabled."
104	)
105
106
107	def app_sessions(app):	1✔
108	app.url_map.strict_slashes = False	1✔
109	app.db = get_SQLAlchemyDriver(config["DB"])	1✔
110
111	# app.db.Session is from SQLAlchemyDriver and uses
112	# SQLAlchemy's sessionmaker. Using scoped_session here ensures
113	# a thread-local db session is created. Effectively the below is expanded to:
114	# app.scoped_session = scoped_session(
115	# sessionmaker(
116	# bind=sqlalchemy.create_engine(config["DB"]),
117	# expire_on_commit=False,
118	# )
119	# )
120	#
121	# From the sqlalchemy docs: "The scoped_session object by default uses
122	# [Python's threading.local() construct] as
123	# storage, so that a single Session is maintained for all who call upon the
124	# scoped_session registry, but only within the scope of a single thread.
125	# Callers who call upon the registry in a different thread get a Session
126	# instance that is local to that other thread."
127	app.scoped_session = scoped_session(app.db.Session)	1✔
128
129	app.session_interface = UserSessionInterface()	1✔
130
131
132	def app_register_blueprints(app):	1✔
133	app.register_blueprint(fence.blueprints.oauth2.blueprint, url_prefix="/oauth2")	1✔
134	app.register_blueprint(fence.blueprints.user.blueprint, url_prefix="/user")	1✔
135
136	creds_blueprint = fence.blueprints.storage_creds.make_creds_blueprint()	1✔
137	app.register_blueprint(creds_blueprint, url_prefix="/credentials")	1✔
138
139	app.register_blueprint(fence.blueprints.admin.blueprint, url_prefix="/admin")	1✔
140	app.register_blueprint(	1✔
141	fence.blueprints.well_known.blueprint, url_prefix="/.well-known"
142	)
143
144	login_blueprint = fence.blueprints.login.make_login_blueprint()	1✔
145	app.register_blueprint(login_blueprint, url_prefix="/login")	1✔
146
147	link_blueprint = fence.blueprints.link.make_link_blueprint()	1✔
148	app.register_blueprint(link_blueprint, url_prefix="/link")	1✔
149
150	google_blueprint = fence.blueprints.google.make_google_blueprint()	1✔
151	app.register_blueprint(google_blueprint, url_prefix="/google")	1✔
152
153	app.register_blueprint(	1✔
154	fence.blueprints.privacy.blueprint, url_prefix="/privacy-policy"
155	)
156
157	app.register_blueprint(fence.blueprints.register.blueprint, url_prefix="/register")	1✔
158	app.register_blueprint(fence.blueprints.ga4gh.blueprint, url_prefix="/ga4gh")	1✔
159
160	fence.blueprints.misc.register_misc(app)	1✔
161
162	@app.route("/")	1✔
163	def root():	1✔
164	"""
165	Register the root URL.
166	"""
167	endpoints = {	×
168	"oauth2 endpoint": "/oauth2",
169	"user endpoint": "/user",
170	"keypair endpoint": "/credentials",
171	}
172	return flask.jsonify(endpoints)	×
173
174	@app.route("/logout")	1✔
175	def logout_endpoint():	1✔
176	root = config.get("BASE_URL", "")	1✔
177	request_next = flask.request.args.get("next", root)	1✔
178	force_era_global_logout = (	1✔
179	flask.request.args.get("force_era_global_logout") == "true"
180	)
181	if request_next.startswith("https") or request_next.startswith("http"):	1✔
182	next_url = request_next	1✔
183	else:
184	next_url = build_redirect_url(config.get("ROOT_URL", ""), request_next)	×
185	if domain(next_url) not in allowed_login_redirects():	1✔
186	raise UserError("invalid logout redirect URL: {}".format(next_url))	1✔
187	return logout(	1✔
188	next_url=next_url, force_era_global_logout=force_era_global_logout
189	)
190
191	@app.route("/jwt/keys")	1✔
192	def public_keys():	1✔
193	"""
194	Return the public keys which can be used to verify JWTs signed by fence.
195
196	The return value should look like this:
197	{
198	"keys": [
199	{
200	"key-01": " ... [public key here] ... "
201	}
202	]
203	}
204	"""
205	return flask.jsonify(	1✔
206	{"keys": [(keypair.kid, keypair.public_key) for keypair in app.keypairs]}
207	)
208
209	@app.route("/metrics")	1✔
210	def metrics_endpoint():	1✔
211	"""
212	WARNING: There is no authz control on this endpoint!
213	In cloud-automation setups, access to this endpoint is blocked at the revproxy level.
214	"""
215	data, content_type = metrics.get_latest_metrics()	1✔
216	return flask.Response(data, content_type=content_type)	1✔
217
218
219	def _check_azure_storage(app):	1✔
220	"""
221	Confirm access to Azure Storage Account and Containers
222	"""
223	azure_creds = config.get("AZ_BLOB_CREDENTIALS", None)	1✔
224
225	# if this is a public bucket, Fence will not try to sign the URL
226	if azure_creds == "*":	1✔
227	return	1✔
228
229	if not azure_creds or azure_creds.strip() == "":	1✔
230	# Azure Blob credentials are not configured.
231	# If you're using Azure Blob Storage set AZ_BLOB_CREDENTIALS to your Azure Blob Storage Connection String.
232	logger.debug(	1✔
233	"Azure Blob credentials are not configured. If you're using Azure Blob Storage, please set AZ_BLOB_CREDENTIALS to your Azure Blob Storage Connection String."
234	)
235	return	1✔
236
237	blob_service_client = BlobServiceClient.from_connection_string(azure_creds)	1✔
238
239	for c in blob_service_client.list_containers():	1✔
240	container_client = blob_service_client.get_container_client(c.name)	1✔
241
242	# check if container exists. If it doesn't exist, log a warning.
243	if container_client.exists() is False:	1✔
244	logger.debug(	1✔
245	f"Unable to access Azure Blob Storage Container {c.name}. You may run into issues resolving orphaned indexed files pointing to this container."
246	)
247	continue	1✔
248
249	# verify that you can check the container properties
250	try:	1✔
251	container_properties = container_client.get_container_properties()	1✔
252	public_access = container_properties["public_access"]	1✔
253	# check container properties
254	logger.debug(	1✔
255	f"Azure Blob Storage Container {c.name} has public access {public_access}"
256	)
257	except ResourceNotFoundError as err:	1✔
258	logger.debug(	1✔
259	f"Unable to access Azure Blob Storage Container {c.name}. You may run into issues resolving orphaned indexed files pointing to this container."
260	)
261	logger.debug(err)	1✔
262
263
264	def _check_buckets_aws_creds_and_region(app):	1✔
265	"""
266	Function to ensure that all s3_buckets have a valid credential.
267	Additionally, if there is no region it will produce a warning
268	then try to fetch and cache the region.
269	"""
270	buckets = config.get("S3_BUCKETS") or {}	1✔
271	aws_creds = config.get("AWS_CREDENTIALS") or {}	1✔
272
273	# check that AWS creds and regions are configured
274	for bucket_name, bucket_details in buckets.items():	1✔
275	cred = bucket_details.get("cred")	1✔
276	region = bucket_details.get("region")	1✔
277	if not cred:	1✔
278	raise ValueError(	×
279	"No cred for S3_BUCKET: {}. cred is required.".format(bucket_name)
280	)
281
282	# if this is a public bucket, Fence will not try to sign the URL
283	# so it won't need to know the region.
284	if cred == "*":	1✔
285	continue	1✔
286
287	if cred not in aws_creds:	1✔
288	raise ValueError(	×
289	"Credential {} for S3_BUCKET {} is not defined in AWS_CREDENTIALS".format(
290	cred, bucket_name
291	)
292	)
293
294	# only require region when we're not specifying an
295	# s3-compatible endpoint URL (ex: no need for region when using cleversafe)
296	if not region and not bucket_details.get("endpoint_url"):	1✔
297	logger.warning(	1✔
298	"WARNING: no region for S3_BUCKET: {}. Providing the region will reduce"
299	" response time and avoid a call to GetBucketLocation which you make lack the AWS ACLs for.".format(
300	bucket_name
301	)
302	)
303	credential = S3IndexedFileLocation.get_credential_to_access_bucket(	1✔
304	bucket_name,
305	aws_creds,
306	config.get("MAX_PRESIGNED_URL_TTL", 3600),
307	app.boto,
308	)
309	if not getattr(app, "boto"):	1✔
310	logger.warning(	×
311	"WARNING: boto not setup for app, probably b/c "
312	"nothing in AWS_CREDENTIALS. Cannot attempt to get bucket "
313	"bucket regions."
314	)
315	return	×
316
317	region = app.boto.get_bucket_region(bucket_name, credential)	1✔
318	config["S3_BUCKETS"][bucket_name]["region"] = region	1✔
319
320	cred = config["PUSH_AUDIT_LOGS_CONFIG"].get("aws_sqs_config", {}).get("aws_cred")	1✔
321	if cred and cred not in aws_creds:	1✔
322	raise ValueError(	×
323	"Credential {} for PUSH_AUDIT_LOGS_CONFIG.aws_sqs_config.aws_cred is not defined in AWS_CREDENTIALS".format(
324	cred
325	)
326	)
327
328	# check that all the configured buckets are in `S3_BUCKETS`
329	bucket_names = config["ALLOWED_DATA_UPLOAD_BUCKETS"] or []	1✔
330	if config["DATA_UPLOAD_BUCKET"]:	1✔
331	bucket_names.append(config["DATA_UPLOAD_BUCKET"])	1✔
332	for bucket_name in bucket_names:	1✔
333	if bucket_name not in buckets:	1✔
334	logger.warning(	×
335	f"Data upload bucket '{bucket_name}' is not configured in 'S3_BUCKETS'"
336	)
337
338
339	def app_config(	1✔
340	app,
341	root_dir=None,
342	config_path=None,
343	file_name=None,
344	):
345	"""
346	Set up the config for the Flask app.
347	"""
348	if root_dir is None:	1✔
349	root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))	×
350
351	# logger.info("Loading settings...")
352	settings_cfg = flask.Config(app.config.root_path)	1✔
353
354	# # dump the settings into the config singleton before loading a configuration file
355	config.update(dict(settings_cfg))	1✔
356
357	# load the configuration file
358	config.load(	1✔
359	config_path=config_path,
360	search_folders=CONFIG_SEARCH_FOLDERS,
361	file_name=file_name,
362	)
363
364	# load all config back into flask app config for now, we should PREFER getting config
365	# directly from the fence config singleton in the code though.
366	app.config.update(**config._configs)	1✔
367
368	_setup_arborist_client(app)	1✔
369	_setup_audit_service_client(app)	1✔
370	_setup_data_endpoint_and_boto(app)	1✔
371	_load_keys(app, root_dir)	1✔
372
373	app.storage_manager = StorageManager(config["STORAGE_CREDENTIALS"], logger=logger)	1✔
374
375	app.debug = config["DEBUG"]	1✔
376	# Following will update logger level, propagate, and handlers
377	get_logger(__name__, log_level="debug" if config["DEBUG"] is True else "info")	1✔
378
379	_setup_oidc_clients(app)	1✔
380
381	with app.app_context():	1✔
382	_check_buckets_aws_creds_and_region(app)	1✔
383	_check_azure_storage(app)	1✔
384
385
386	def _setup_data_endpoint_and_boto(app):	1✔
387	if "AWS_CREDENTIALS" in config and len(config["AWS_CREDENTIALS"]) > 0:	1✔
388	creds = config["AWS_CREDENTIALS"]	1✔
389	buckets = config.get("S3_BUCKETS", {})	1✔
390	app.boto = BotoManager(creds, buckets, logger=logger)	1✔
391	app.register_blueprint(fence.blueprints.data.blueprint, url_prefix="/data")	1✔
392
393
394	def _load_keys(app, root_dir):	1✔
395	if root_dir is None:	1✔
396	root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))	×
397
398	app.keypairs = keys.load_keypairs(os.path.join(root_dir, "keys"))	1✔
399
400	app.jwt_public_keys = {	1✔
401	config["BASE_URL"]: OrderedDict(
402	[(str(keypair.kid), str(keypair.public_key)) for keypair in app.keypairs]
403	)
404	}
405
406
407	def _setup_oidc_clients(app):	1✔
408	configured_idps = config.get("OPENID_CONNECT", {})	1✔
409
410	clean_idps = [idp.lower().replace(" ", "") for idp in configured_idps]	1✔
411	if len(clean_idps) != len(set(clean_idps)):	1✔
412	raise ValueError(	×
413	f"Some IDPs configured in OPENID_CONNECT are not unique once they are lowercased and spaces are removed: {clean_idps}"
414	)
415
416	for idp in set(configured_idps.keys()):	1✔
417	logger.info(f"Setting up OIDC client for {idp}")	1✔
418	settings = configured_idps[idp]	1✔
419	if idp == "google":	1✔
420	app.google_client = GoogleOauth2Client(	1✔
421	settings,
422	HTTP_PROXY=config.get("HTTP_PROXY"),
423	logger=logger,
424	)
425	elif idp == "orcid":	1✔
426	app.orcid_client = OrcidOauth2Client(	1✔
427	settings,
428	HTTP_PROXY=config.get("HTTP_PROXY"),
429	logger=logger,
430	)
431	elif idp == "ras":	1✔
432	app.ras_client = RASOauth2Client(	1✔
433	settings,
434	HTTP_PROXY=config.get("HTTP_PROXY"),
435	logger=logger,
436	)
437	elif idp == "synapse":	1✔
438	app.synapse_client = SynapseOauth2Client(	1✔
439	settings, HTTP_PROXY=config.get("HTTP_PROXY"), logger=logger
440	)
441	elif idp == "microsoft":	1✔
442	app.microsoft_client = MicrosoftOauth2Client(	1✔
443	settings,
444	HTTP_PROXY=config.get("HTTP_PROXY"),
445	logger=logger,
446	)
447	elif idp == "okta":	1✔
448	app.okta_client = OktaOauth2Client(	1✔
449	settings,
450	HTTP_PROXY=config.get("HTTP_PROXY"),
451	logger=logger,
452	)
453	elif idp == "cognito":	1✔
454	app.cognito_client = CognitoOauth2Client(	1✔
455	settings, HTTP_PROXY=config.get("HTTP_PROXY"), logger=logger
456	)
457	elif idp == "cilogon":	1✔
458	app.cilogon_client = CilogonOauth2Client(	1✔
459	settings,
460	HTTP_PROXY=config.get("HTTP_PROXY"),
461	logger=logger,
462	)
463	elif idp == "fence":	1✔
464	# https://docs.authlib.org/en/latest/client/flask.html#flask-client
465	app.fence_client = OAuthClient(app)	1✔
466	# https://docs.authlib.org/en/latest/client/frameworks.html
467	app.fence_client.register(**settings)	1✔
468	else: # generic OIDC implementation
469	if hasattr(app, "arborist"):	1✔
470	app_arborist = app.arborist	1✔
471	else:
472	app_arborist = None	×
473	client = Oauth2ClientBase(	1✔
474	settings=settings,
475	logger=logger,
476	HTTP_PROXY=config.get("HTTP_PROXY"),
477	idp=settings.get("name") or idp.title(),
478	arborist=app_arborist,
479	)
480	clean_idp = idp.lower().replace(" ", "")	1✔
481	setattr(app, f"{clean_idp}_client", client)	1✔
482
483
484	def _setup_arborist_client(app):	1✔
485	if app.config.get("ARBORIST"):	1✔
486	app.arborist = ArboristClient(	1✔
487	arborist_base_url=config["ARBORIST"],
488	timeout=app.config.get("ARBORIST_TIMEOUT", 30),
489	)
490	else:
491	logger.info("Arborist not configured")	×
492	app.arborist = None	×
493
494
495	def _setup_audit_service_client(app):	1✔
496	# Initialize the client regardless of whether audit logs are enabled. This
497	# allows us to call `app.audit_service_client.create_x_log()` from
498	# anywhere without checking if audit logs are enabled. The client
499	# checks that for us.
500	service_url = app.config.get("AUDIT_SERVICE") or urljoin(	1✔
501	app.config["BASE_URL"], "/audit"
502	)
503	app.audit_service_client = AuditServiceClient(	1✔
504	service_url=service_url, logger=logger
505	)
506
507
508	@app.errorhandler(Exception)	1✔
509	def handle_error(error):	1✔
510	"""
511	Register an error handler for general exceptions.
512	"""
513	return get_error_response(error)	1✔
514
515
516	@app.before_request	1✔
517	def check_csrf():	1✔
518	has_auth = "Authorization" in flask.request.headers	1✔
519	no_username = not flask.session.get("username")	1✔
520	if has_auth or no_username:	1✔
521	return	1✔
522	if not config.get("ENABLE_CSRF_PROTECTION", True):	1✔
523	return	1✔
524	if flask.request.method != "GET":	×
525	try:	×
526	csrf_header = flask.request.headers.get("x-csrf-token")	×
527	csrf_formfield = flask.request.form.get("csrf_token")	×
528	# validate_csrf checks the input (a signed token) against the raw
529	# token stored in session["csrf_token"].
530	# (session["csrf_token"] is managed by flask-wtf.)
531	# To pass CSRF check, there must exist EITHER an x-csrf-token header
532	# OR a csrf_token form field that matches the token in the session.
533	assert (	×
534	csrf_header
535	and validate_csrf(csrf_header) is None
536	or csrf_formfield
537	and validate_csrf(csrf_formfield) is None
538	)
539
540	referer = flask.request.headers.get("referer")	×
541	assert referer, "Referer header missing"	×
542	logger.debug("HTTP REFERER " + str(referer))	×
543	except Exception as e:	×
544	raise UserError("CSRF verification failed: {}. Request aborted".format(e))	×
545
546
547	@app.teardown_appcontext	1✔
548	def remove_scoped_session(args, *kwargs):	1✔
549	if hasattr(app, "scoped_session"):	1✔
550	try:	1✔
551	app.scoped_session.remove()	1✔
552	except Exception as exc:	1✔
553	logger.warning(f"could not remove app.scoped_session. Error: {exc}")	1✔

uc-cdis / fence / 25459641641

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous