#8755

Committed 10 Dec 2024 09:20PM UTC coverage: 78.792% (-0.008%) from 78.8%

Build # #8755

Build Type

push

coveralls-python

Committed by

web-flow

Commit Message

Fix organization delete - ensure there is a chance to confirm and that it works in one step. (#4851)

* new confirm modal for removing organizations, rearrange inventory deletion so as to not try to remove org before the org is empty

* linting

* update translations and css

* lint

---------

Co-authored-by: Alex Swindler <alex.swindler@nrel.gov>
Co-authored-by: Katherine Fleming <2205659+kflemin@users.noreply.github.com>

Run Details

25 of 38 new or added lines in 2 files covered. (65.79%)

4 existing lines in 1 file now uncovered.

18472 of 23444 relevant lines covered (78.79%)

0.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

61.05

/seed/tasks.py

"""
SEED Platform (TM), Copyright (c) Alliance for Sustainable Energy, LLC, and other contributors.
See also https://github.com/SEED-platform/seed/blob/main/LICENSE.md
"""

import itertools
import math
import sys
from datetime import datetime
from random import randint

import pytz
from celery import chord, shared_task
from celery.utils.log import get_task_logger
from django.conf import settings
from django.core.mail import send_mail
from django.db import transaction
from django.db.models import Q
from django.template import Context, Template, loader
from django.urls import reverse_lazy
from django.utils.encoding import force_bytes
from django.utils.http import urlsafe_base64_encode

from seed.audit_template.audit_template import AuditTemplate
from seed.data_importer.tasks import hash_state_object
from seed.decorators import lock_and_track
from seed.lib.mcm.utils import batch
from seed.lib.progress_data.progress_data import ProgressData
from seed.lib.superperms.orgs.models import Organization
from seed.models import (
    DATA_STATE_MATCHING,
    Column,
    ColumnMapping,
    Cycle,
    DerivedColumn,
    Property,
    PropertyState,
    PropertyView,
    SalesforceConfig,
    TaxLot,
    TaxLotState,
    TaxLotView,
)
from seed.utils.salesforce import auto_sync_salesforce_properties

logger = get_task_logger(__name__)


def invite_new_user_to_seed(domain, email_address, token, user_pk, first_name):
    """Send invitation email to newly created user from the landing page.
    NOTE: this function is only used on the landing page because the user has not been assigned an organization
    domain -- The domain name of the running seed instance
    email_address -- The address to send the invitation to
    token -- generated by Django's default_token_generator
    user_pk -- primary key for this user record
    first_name -- First name of the new user
    new_user

    Returns: nothing
    """
    signup_url = reverse_lazy("landing:activate", kwargs={"uidb64": urlsafe_base64_encode(force_bytes(user_pk)), "token": token})

    context = {"email": email_address, "domain": domain, "protocol": settings.PROTOCOL, "first_name": first_name, "signup_url": signup_url}

    subject = "New SEED account"
    email_body = loader.render_to_string("seed/account_create_email.html", context)
    send_mail(subject, email_body, settings.SERVER_EMAIL, [email_address])
    try:
        bcc_address = settings.SEED_ACCOUNT_CREATION_BCC
        new_subject = f"{subject} ({email_address})"
        send_mail(new_subject, email_body, settings.SERVER_EMAIL, [bcc_address])
    except AttributeError:
        pass


@shared_task
def invite_to_seed(domain, email_address, token, organization, user_pk, first_name):
    """Send invitation email to newly created user.

    domain -- The domain name of the running seed instance
    email_address -- The address to send the invitation to
    token -- generated by Django's default_token_generator
    organization --  the organization user was invited to
    user_pk -- primary key for this user record
    first_name -- First name of the new user

    Returns: nothing
    """
    sign_up_url = Template("{{protocol}}://{{domain}}{{sign_up_url}}").render(
        Context(
            {
                "protocol": settings.PROTOCOL,
                "domain": domain,
                "sign_up_url": reverse_lazy(
                    "landing:signup", kwargs={"uidb64": urlsafe_base64_encode(force_bytes(user_pk)), "token": token}
                ),
            }
        )
    )

    content = Template(organization.new_user_email_content).render(Context({"first_name": first_name, "sign_up_link": sign_up_url}))

    body = Template("{{content}}\n\n{{signature}}").render(
        Context({"content": content, "signature": organization.new_user_email_signature})
    )

    send_mail(organization.new_user_email_subject, body, organization.new_user_email_from, [email_address])
    try:
        bcc_address = settings.SEED_ACCOUNT_CREATION_BCC
        new_subject = f"{organization.new_user_email_subject} ({email_address})"
        send_mail(new_subject, body, organization.new_user_email_from, [bcc_address])
    except AttributeError:
        pass


@shared_task
def invite_to_organization(domain, new_user, requested_by, new_org):
    """Send invitation to a newly created organization.

    domain -- The domain name of the running seed instance
    email_address -- The address to send the invitation to
    token -- generated by Django's default_token_generator
    user_pk --primary key for this user record
    first_name -- First name of the new user

    Returns: nothing
    """
    context = {
        "new_user": new_user,
        "first_name": new_user.first_name,
        "domain": domain,
        "protocol": settings.PROTOCOL,
        "new_org": new_org,
        "requested_by": requested_by,
    }

    subject = "Your SEED account has been added to an organization"
    email_body = loader.render_to_string("seed/account_org_added.html", context)
    send_mail(subject, email_body, settings.SERVER_EMAIL, [new_user.email])
    try:
        bcc_address = settings.SEED_ACCOUNT_CREATION_BCC
        new_subject = f"{subject} ({new_user.email})"
        send_mail(new_subject, email_body, settings.SERVER_EMAIL, [bcc_address])
    except AttributeError:
        pass


def send_salesforce_error_log(org_pk, errors):
    """send salesforce error log to logging email when errors are encountered during scheduled sync"""
    sf_conf = SalesforceConfig.objects.get(organization_id=org_pk)
    org = Organization.objects.get(pk=org_pk)

    if sf_conf.logging_email:
        context = {"organization_name": org.name, "errors": errors}

        subject = "Salesforce Automatic Update Errors"
        email_body = loader.render_to_string("seed/salesforce_update_errors.html", context)
        send_mail(subject, email_body, settings.SERVER_EMAIL, [sf_conf.logging_email])


@shared_task
@lock_and_track
def delete_organization_and_inventory(org_pk, prog_key=None):
    """Deletes all associated inventory and the containing org"""

    progress_data = (
        ProgressData.from_key(prog_key) if prog_key else ProgressData(func_name="delete_organization_and_inventory", unique_id=org_pk)
    )
    _evaluate_delete_organization_and_inventory.subtask((progress_data.key, org_pk, True)).apply_async()
    return progress_data.result()


@shared_task
def _evaluate_delete_organization_and_inventory(prog_key, org_pk, delete_org=False):
    "check for inventory, delete it if it exists, then pass to finish function to delete the organization"
    chunk_size = 100

    progress_data = ProgressData.from_key(prog_key)

    property_ids = list(Property.objects.filter(organization_id=org_pk).values_list("id", flat=True))
    property_state_ids = list(PropertyState.objects.filter(organization_id=org_pk).values_list("id", flat=True))
    taxlot_ids = list(TaxLot.objects.filter(organization_id=org_pk).values_list("id", flat=True))
    taxlot_state_ids = list(TaxLotState.objects.filter(organization_id=org_pk).values_list("id", flat=True))

    total = len(property_ids) + len(property_state_ids) + len(taxlot_ids) + len(taxlot_state_ids)
    progress_data.total = total / float(chunk_size) + 1
    progress_data.data["completed_records"] = 0
    progress_data.data["total_records"] = total
    progress_data.save()

    if total > 0:
        for chunk_ids in batch(property_ids, chunk_size):
            _delete_organization_children(chunk_ids, Property, progress_data.key)
        for chunk_ids in batch(property_state_ids, chunk_size):
            _delete_organization_children(chunk_ids, PropertyState, progress_data.key)
        for chunk_ids in batch(taxlot_ids, chunk_size):
            _delete_organization_children(chunk_ids, TaxLot, progress_data.key)
        for chunk_ids in batch(taxlot_state_ids, chunk_size):
            _delete_organization_children(chunk_ids, TaxLotState, progress_data.key)
    else:
        progress_data.step()

    if delete_org:
        DerivedColumn.objects.filter(organization_id=org_pk).delete()
        Organization.objects.get(pk=org_pk).delete()
        # TODO: Delete measures in BRICR branch


@shared_task
def _delete_organization_children(chunk_ids, class_name, prog_key):
    class_name.objects.filter(id__in=chunk_ids).delete()
    progress_data = ProgressData.from_key(prog_key)
    progress_data.step_with_counter()


@shared_task
def _finish_delete_column(column_id, prog_key):
    # Delete all mappings from raw column names to the mapped column, then delete the mapped column
    column = Column.objects.get(id=column_id)
    ColumnMapping.objects.filter(column_mapped=column).delete()
    column.delete()

    progress_data = ProgressData.from_key(prog_key)
    return progress_data.finish_with_success(f'Removed {column.column_name} from {progress_data.data["total_records"]} records')


@shared_task
@lock_and_track
def delete_organization_inventory(org_pk, prog_key=None, chunk_size=100, *args, **kwargs):
    """Deletes all properties & taxlots within an organization."""
    sys.setrecursionlimit(5000)  # default is 1000
    progress_data = (
        ProgressData.from_key(prog_key) if prog_key else ProgressData(func_name="delete_organization_inventory", unique_id=org_pk)
    )
    _evaluate_delete_organization_and_inventory.subtask((progress_data.key, org_pk, False)).apply_async()
    sys.setrecursionlimit(1000)  # default is 1000
    return progress_data.result()


@shared_task
@lock_and_track
def delete_organization_cycle(cycle_pk, organization_pk, prog_key=None, chunk_size=100, *args, **kwargs):
    """Deletes an organization's cycle.

    This must be an async task b/c a cascading deletion can require the removal
    of many *States associated with an ImportFile, overwhelming the server.

    :param cycle_pk: int
    :param prog_key: str
    :param chunk_size: int
    :return: dict, from ProgressData.result()
    """
    progress_data = ProgressData.from_key(prog_key) if prog_key else ProgressData(func_name="delete_organization_cycle", unique_id=cycle_pk)

    has_inventory = PropertyView.objects.filter(cycle_id=cycle_pk).exists() or TaxLotView.objects.filter(cycle_id=cycle_pk).exists()
    if has_inventory:
        progress_data.finish_with_error("All PropertyView and TaxLotViews linked to the Cycle must be removed")
        return progress_data.result()

    property_state_ids = PropertyState.objects.filter(import_file__cycle_id=cycle_pk).values_list("id", flat=True)
    tax_lot_state_ids = TaxLotState.objects.filter(import_file__cycle_id=cycle_pk).values_list("id", flat=True)
    progress_data.total = (len(property_state_ids) + len(tax_lot_state_ids)) / chunk_size
    progress_data.save()

    tasks = []
    for chunk_ids in batch(property_state_ids, chunk_size):
        tasks.append(_delete_organization_property_state_chunk.si(chunk_ids, progress_data.key, organization_pk))
    for chunk_ids in batch(tax_lot_state_ids, chunk_size):
        tasks.append(_delete_organization_taxlot_state_chunk.si(chunk_ids, progress_data.key, organization_pk))

    chord(tasks, interval=15)(_finish_delete_cycle.si(cycle_pk, progress_data.key))

    return progress_data.result()


@shared_task
def _finish_delete_cycle(cycle_id, prog_key):
    # Finally delete the cycle
    cycle = Cycle.objects.get(id=cycle_id)
    cycle.delete()

    progress_data = ProgressData.from_key(prog_key)
    return progress_data.finish_with_success(f"Removed {cycle.name}")


@shared_task
@lock_and_track
def delete_organization_column(column_pk, org_pk, prog_key=None, chunk_size=100, *args, **kwargs):
    """Deletes an extra_data column from all merged property/taxlot states."""
    progress_data = (
        ProgressData.from_key(prog_key) if prog_key else ProgressData(func_name="delete_organization_column", unique_id=column_pk)
    )

    _evaluate_delete_organization_column.subtask((column_pk, org_pk, progress_data.key, chunk_size)).apply_async()

    return progress_data.result()


@shared_task
@lock_and_track
def update_multiple_columns(key, table_name, org_pk, changes, prog_key=None):
    """Updates several columns and optionally rehashes if need be."""

    progress_data = ProgressData.from_key(prog_key) if prog_key else ProgressData(func_name="update_multiple_columns", unique_id=key)
    _evaluate_update_multiple_columns.subtask((progress_data.key, table_name, org_pk, changes)).apply_async()
    return progress_data.result()


@shared_task
def _evaluate_update_multiple_columns(prog_key, table_name, org_pk, changes):
    """Update columns, then check for required rehash - and rehash states if need be"""

    chunk_size = 100

    rehashed_columns = []
    for key in changes:
        c = Column.objects.get(pk=key)
        for attribute in changes[key]:
            if attribute == "is_excluded_from_hash":
                rehashed_columns.append(c)
            setattr(c, attribute, changes[key][attribute])
        c.save()

    progress_data = ProgressData.from_key(prog_key)

    if len(rehashed_columns) == 0:
        total = len(changes.keys())
        progress_data.total = total / float(chunk_size) + 1
        progress_data.data["completed_records"] = total
        progress_data.data["total_records"] = total
        progress_data.save()
    else:
        query = _build_property_query_for_rehashed_columns(org_pk, rehashed_columns)

        ids = []

        if table_name == "PropertyState":
            ids = PropertyState.objects.filter(query).values_list("id", flat=True)
        elif table_name == "TaxLotState":
            ids = TaxLotState.objects.filter(query).values_list("id", flat=True)

        total = len(ids)
        progress_data.total = total / float(chunk_size) + 1
        progress_data.data["completed_records"] = 0
        progress_data.data["total_records"] = total
        progress_data.save()

        for chunk_ids in batch(ids, chunk_size):
            _rehash_state_chunk(chunk_ids, table_name, progress_data.key)

    _finish_update_multiple_columns(changes, progress_data.key)


@shared_task
def _build_property_query_for_rehashed_columns(org_pk, rehashed_columns):
    query = Q()
    query.add(Q(data_state=DATA_STATE_MATCHING), Q.AND)
    query.add(Q(organization_id=org_pk), Q.AND)
    or_query = Q()
    fields = []
    extra_fields = []
    for column in rehashed_columns:
        if column.is_extra_data:
            extra_fields.append(column.column_name)
        else:
            fields.append(column.column_name)

    for field in fields:
        or_query.add(Q(**{field + "__isnull": False}), Q.OR)
    if len(extra_fields) > 0:
        or_query.add(Q(extra_data__has_any_key=extra_fields), Q.OR)

    query.add(or_query, Q.AND)
    return query


@shared_task
def _rehash_state_chunk(chunk_ids, table_name, prog_key):
    if table_name == "PropertyState":
        states = PropertyState.objects.filter(id__in=chunk_ids)
    else:
        states = TaxLotState.objects.filter(id__in=chunk_ids)
    with transaction.atomic():
        for state in states:
            state.hash_object = hash_state_object(state)
            state.save(update_fields=["hash_object"])

    progress_data = ProgressData.from_key(prog_key)
    progress_data.step_with_counter()


@shared_task
def _finish_update_multiple_columns(changes, prog_key):
    progress_data = ProgressData.from_key(prog_key)
    if progress_data.data["total_records"] == len(changes):
        return progress_data.finish_with_success(f"Updated {len(changes.keys())} columns.")
    else:
        return progress_data.finish_with_success(
            f'Updated {len(changes.keys())} columns.  Rebuilt {progress_data.data["total_records"]} records'
        )


@shared_task
def _evaluate_delete_organization_column(column_pk, org_pk, prog_key, chunk_size, *args, **kwargs):
    """Find -States with column to be deleted"""
    column = Column.objects.get(id=column_pk, organization_id=org_pk)

    ids = []

    if column.table_name == "PropertyState":
        ids = PropertyState.objects.filter(
            organization_id=org_pk, data_state=DATA_STATE_MATCHING, extra_data__has_key=column.column_name
        ).values_list("id", flat=True)
    elif column.table_name == "TaxLotState":
        ids = TaxLotState.objects.filter(
            organization_id=org_pk, data_state=DATA_STATE_MATCHING, extra_data__has_key=column.column_name
        ).values_list("id", flat=True)

    progress_data = ProgressData.from_key(prog_key)
    total = len(ids)
    progress_data.total = total / float(chunk_size) + 1
    progress_data.data["completed_records"] = 0
    progress_data.data["total_records"] = total
    progress_data.save()

    for chunk_ids in batch(ids, chunk_size):
        _delete_organization_column_chunk(chunk_ids, column.column_name, column.table_name, progress_data.key)

    _finish_delete_column(column_pk, progress_data.key)


def _delete_organization_column_chunk(chunk_ids, column_name, table_name, prog_key, *args, **kwargs):
    """updates a list of ``chunk_ids`` and increments the cache"""

    if table_name == "PropertyState":
        states = PropertyState.objects.filter(id__in=chunk_ids)
    else:
        states = TaxLotState.objects.filter(id__in=chunk_ids)

    with transaction.atomic():
        for state in states:
            del state.extra_data[column_name]
            state.save(update_fields=["extra_data", "hash_object"])

    progress_data = ProgressData.from_key(prog_key)
    progress_data.step_with_counter()


@shared_task
def _delete_organization_property_chunk(del_ids, prog_key, org_pk, *args, **kwargs):
    """deletes a list of ``del_ids`` and increments the cache"""
    Property.objects.filter(organization_id=org_pk, pk__in=del_ids).delete()
    progress_data = ProgressData.from_key(prog_key)
    progress_data.step()


@shared_task
def _delete_organization_property_state_chunk(del_ids, prog_key, org_pk, *args, **kwargs):
    """deletes a list of ``del_ids`` and increments the cache"""
    PropertyState.objects.filter(pk__in=del_ids).delete()
    progress_data = ProgressData.from_key(prog_key)
    progress_data.step()


@shared_task
def _delete_organization_taxlot_chunk(del_ids, prog_key, org_pk, *args, **kwargs):
    """deletes a list of ``del_ids`` and increments the cache"""
    TaxLot.objects.filter(organization_id=org_pk, pk__in=del_ids).delete()
    progress_data = ProgressData.from_key(prog_key)
    progress_data.step()


@shared_task
def _delete_organization_taxlot_state_chunk(del_ids, prog_key, org_pk, *args, **kwargs):
    """deletes a list of ``del_ids`` and increments the cache"""
    TaxLotState.objects.filter(organization_id=org_pk, pk__in=del_ids).delete()
    progress_data = ProgressData.from_key(prog_key)
    progress_data.step()


@shared_task
def sync_salesforce(org_id):
    status, messages = auto_sync_salesforce_properties(org_id)
    if not status:
        # send email with errors
        send_salesforce_error_log(org_id, messages)


@shared_task
def sync_audit_template(org_id):
    try:
        org = Organization.objects.get(id=org_id)
    except Organization.DoesNotExist:
        return

    if not org.audit_template_city_id:
        return

    at = AuditTemplate(org_id)
    at.batch_get_city_submission_xml([])


@shared_task
def set_update_to_now(property_view_ids, taxlot_view_ids, progress_key):
    now = datetime.now(pytz.UTC)
    progress_data = ProgressData.from_key(progress_key)
    progress_data.total = 100
    id_count = len(property_view_ids) + len(taxlot_view_ids)
    batch_size = math.ceil(id_count / 100)

    property_views = PropertyView.objects.filter(id__in=property_view_ids).prefetch_related("state", "property")
    taxlot_views = TaxLotView.objects.filter(id__in=taxlot_view_ids).prefetch_related("state", "taxlot")

    with transaction.atomic():
        for idx, view in enumerate(itertools.chain(property_views, taxlot_views)):
            view.state.updated = now
            view.state.save()

            if isinstance(view, PropertyView):
                view.property.update = now
                view.property.save()
            else:
                view.taxlot.update = now
                view.taxlot.save()

            if batch_size > 0 and idx % batch_size == 0:
                progress_data.step(f"Refreshing ({idx}/{id_count})")

    progress_data.finish_with_success()
    return progress_data.result()["progress"]


@shared_task
def update_state_derived_data(property_state_ids=[], taxlot_state_ids=[], derived_column_ids=[]):
    progress_data = ProgressData(func_name="update_derived_data", unique_id=randint(10000, 99999))
    progress_data.total = len(property_state_ids) + len(taxlot_state_ids)
    progress_data.save()
    progress_key = progress_data.key

    chunk_size = 100

    derived_columns = DerivedColumn.objects.filter(id__in=derived_column_ids)
    property_derived_column_ids = list(derived_columns.filter(inventory_type=DerivedColumn.PROPERTY_TYPE).values_list("id", flat=True))
    taxlot_derived_column_ids = list(derived_columns.filter(inventory_type=DerivedColumn.TAXLOT_TYPE).values_list("id", flat=True))

    tasks = []
    for chunk_ids in batch(property_state_ids, chunk_size):
        tasks.append(_update_property_state_derived_data_chunk.si(progress_key, chunk_ids, property_derived_column_ids))
    for chunk_ids in batch(taxlot_state_ids, chunk_size):
        tasks.append(_update_taxlot_state_derived_data_chunk.si(progress_key, chunk_ids, taxlot_derived_column_ids))

    chord(tasks, interval=15)(_finish_update_state_derived_data.si(progress_key, property_derived_column_ids + taxlot_derived_column_ids))

    return progress_data.result()


@shared_task
def _update_property_state_derived_data_chunk(progress_key, property_state_ids=[], derived_column_ids=[]):
    progress_data = ProgressData.from_key(progress_key)

    states = PropertyState.objects.filter(id__in=property_state_ids)
    derived_columns = DerivedColumn.objects.filter(id__in=derived_column_ids)

    for state in states:
        for derived_column in derived_columns:
            state.derived_data[derived_column.name] = derived_column.evaluate(state)
        state.save()
        progress_data.step()


@shared_task
def _update_taxlot_state_derived_data_chunk(progress_key, taxlot_state_ids=[], derived_column_ids=[]):
    progress_data = ProgressData.from_key(progress_key)

    states = TaxLotState.objects.filter(id__in=taxlot_state_ids)
    derived_columns = DerivedColumn.objects.filter(id__in=derived_column_ids)

    for state in states:
        for derived_column in derived_columns:
            state.derived_data[derived_column.name] = derived_column.evaluate(state)
        state.save()
        progress_data.step()


@shared_task
def _finish_update_state_derived_data(progress_key, derived_column_ids):
    derived_columns = DerivedColumn.objects.filter(id__in=derived_column_ids)
    Column.objects.filter(derived_column__in=derived_columns).update(is_updating=False)

    progress_data = ProgressData.from_key(progress_key)
    progress_data.finish_with_success("Updated Derived Data")

SEED-platform / seed / #8755

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous