diff --git a/config/runtime.exs b/config/runtime.exs index 5a85deb255e9..2852e18bcd30 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -855,9 +855,10 @@ cloud_cron = [ # {"0 7 * * *", Plausible.Workers.ScanInactiveTeams}, # Daily at 8 {"0 8 * * *", Plausible.Workers.AcceptTrafficUntil}, - # Daily at 9, after AcceptTrafficUntil + # Weekdays at 9, after AcceptTrafficUntil - no deletion notices go out on + # weekends; anything due Sat/Sun is simply picked up on Monday instead # TODO: enable - # {"0 9 * * *", Plausible.Workers.SendDeletionNotifications}, + # {"0 9 * * 1-5", Plausible.Workers.SendDeletionNotifications}, # Daily at 10, after SendDeletionNotifications # TODO: enable # {"0 10 * * *", Plausible.Workers.ExecuteTeamDeletions}, diff --git a/lib/plausible/team_deletion_schedules.ex b/lib/plausible/team_deletion_schedules.ex index 0116c14eaa0c..300bff8f7df4 100644 --- a/lib/plausible/team_deletion_schedules.ex +++ b/lib/plausible/team_deletion_schedules.ex @@ -352,17 +352,25 @@ defmodule Plausible.TeamDeletionSchedules do deletion_date = DeletionSchedule.deletion_date(category, expiry_date) steady_state_first_notice_due_date = DeletionSchedule.first_notice_due_date(deletion_date) - is_backlog? = Date.before?(steady_state_first_notice_due_date, today) + backlog? = Date.before?(steady_state_first_notice_due_date, today) + + first_notice_due_date = + if backlog? do + # Spread backlog rows across the release window rather than piling + # them all onto the day they're first discovered. + offset = rem(candidate.team_id, DeletionSchedule.backlog_release_window_days()) + Date.add(today, offset) + else + steady_state_first_notice_due_date + end %{ team_id: candidate.team_id, category: category, expiry_date: expiry_date, deletion_date: deletion_date, - # Backlog rows will be spread-out eventually. - # `today` is temporary for now. - first_notice_due_date: if(is_backlog?, do: today, else: steady_state_first_notice_due_date), - is_backlog: is_backlog?, + first_notice_due_date: first_notice_due_date, + is_backlog: backlog?, status: :scheduled, inserted_at: now, updated_at: now diff --git a/lib/plausible/teams/deletion_schedule.ex b/lib/plausible/teams/deletion_schedule.ex index 0b88c376c680..ce18bb8e642e 100644 --- a/lib/plausible/teams/deletion_schedule.ex +++ b/lib/plausible/teams/deletion_schedule.ex @@ -10,7 +10,7 @@ defmodule Plausible.Teams.DeletionSchedule do @reminder_before_deletion_days 5 @backlog_deletion_offset_days 30 - @backlog_release_window_days 30 + @backlog_release_window_days 60 # how many domains to include in notifications @notification_site_list_limit 3 diff --git a/lib/plausible/telemetry/plausible_metrics.ex b/lib/plausible/telemetry/plausible_metrics.ex index 524e94f9419c..11f741a8782e 100644 --- a/lib/plausible/telemetry/plausible_metrics.ex +++ b/lib/plausible/telemetry/plausible_metrics.ex @@ -240,6 +240,35 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do event_name: Plausible.Workers.ClickhouseCleanSites.telemetry_partitions_event(), measurement: :count, tags: [:table] + ), + last_value( + metric_prefix ++ [:scan_inactive_teams, :run, :created_count], + event_name: Plausible.Workers.ScanInactiveTeams.telemetry_run_event(), + measurement: :created + ), + last_value( + metric_prefix ++ [:unsnooze_team_deletions, :run, :count], + event_name: Plausible.Workers.UnsnoozeTeamDeletions.telemetry_run_event(), + measurement: :count + ), + counter( + metric_prefix ++ [:send_deletion_notifications, :run, :total], + event_name: Plausible.Workers.SendDeletionNotifications.telemetry_run_event(), + tags: [:stage, :outcome, :category] + ), + counter( + metric_prefix ++ [:execute_team_deletions, :run, :total], + event_name: Plausible.Workers.ExecuteTeamDeletions.telemetry_run_event(), + tags: [:outcome, :category] + ), + distribution( + metric_prefix ++ [:execute_team_deletions, :sites_deleted], + event_name: Plausible.Workers.ExecuteTeamDeletions.telemetry_sites_deleted_event(), + reporter_options: [ + buckets: [1, 2, 5, 10, 25, 50, 100, 250, 500, 1_000] + ], + measurement: :count, + tags: [:category] ) ] |> Enum.concat(persistor_metrics(metric_prefix)) diff --git a/lib/workers/execute_team_deletions.ex b/lib/workers/execute_team_deletions.ex index 65ea4d4d271d..9da6d3bb763e 100644 --- a/lib/workers/execute_team_deletions.ex +++ b/lib/workers/execute_team_deletions.ex @@ -16,6 +16,12 @@ defmodule Plausible.Workers.ExecuteTeamDeletions do alias Plausible.TeamDeletionSchedules alias Plausible.Teams + @spec telemetry_run_event() :: [atom()] + def telemetry_run_event(), do: [:plausible, :execute_team_deletions, :run] + + @spec telemetry_sites_deleted_event() :: [atom()] + def telemetry_sites_deleted_event(), do: [:plausible, :execute_team_deletions, :sites_deleted] + @impl Oban.Worker def perform(_job, today \\ Date.utc_today()) do for schedule <- TeamDeletionSchedules.due_for_deletion(today) do @@ -23,6 +29,8 @@ defmodule Plausible.Workers.ExecuteTeamDeletions do if TeamDeletionSchedules.cancel_for_team(team) == :no_schedule do execute(schedule, team) + else + report(schedule, :cancelled) end end @@ -31,11 +39,26 @@ defmodule Plausible.Workers.ExecuteTeamDeletions do defp execute(schedule, team) do Repo.transaction(fn -> - for site <- Teams.owned_sites(team) do + sites = Teams.owned_sites(team) + + for site <- sites do Plausible.Site.Removal.run(site, reason: schedule.category) end TeamDeletionSchedules.mark_completed(schedule, report_if_invalid?: true) + + report(schedule, :executed) + + :telemetry.execute(telemetry_sites_deleted_event(), %{count: length(sites)}, %{ + category: schedule.category + }) end) end + + defp report(schedule, outcome) do + :telemetry.execute(telemetry_run_event(), %{count: 1}, %{ + outcome: outcome, + category: schedule.category + }) + end end diff --git a/lib/workers/send_deletion_notifications.ex b/lib/workers/send_deletion_notifications.ex index 8dea14df1b72..bc4ccedd5c5a 100644 --- a/lib/workers/send_deletion_notifications.ex +++ b/lib/workers/send_deletion_notifications.ex @@ -14,6 +14,9 @@ defmodule Plausible.Workers.SendDeletionNotifications do alias Plausible.Teams alias Plausible.Teams.DeletionSchedule + @spec telemetry_run_event() :: [atom()] + def telemetry_run_event(), do: [:plausible, :send_deletion_notifications, :run] + @impl Oban.Worker def perform(_job, today \\ Date.utc_today()) do # Anchor to `today` @@ -27,40 +30,73 @@ defmodule Plausible.Workers.SendDeletionNotifications do defp send_first_notices(today, now) do for schedule <- TeamDeletionSchedules.due_for_first_notice(today) do - team = schedule.team - - if TeamDeletionSchedules.cancel_for_team(team) == :no_schedule do - summary = sites_summary(team) - - for recipient <- team.owners ++ team.billing_members do - recipient - |> PlausibleWeb.Email.deletion_full_notice_email(team, schedule, summary) - |> Plausible.Mailer.send() - end + send_first_notice(schedule, now) + end + end - TeamDeletionSchedules.mark_first_notice_sent(schedule, now: now, report_if_invalid?: true) + defp send_first_notice(schedule, now) do + team = schedule.team + + if TeamDeletionSchedules.cancel_for_team(team) == :no_schedule do + # Finalize the schedule (which, for backlog rows, anchors deletion_date + # to `now`) before composing the email, so the date we tell the + # customer matches the date we actually persist. + case TeamDeletionSchedules.mark_first_notice_sent(schedule, + now: now, + report_if_invalid?: true + ) do + {:ok, schedule} -> + summary = sites_summary(team) + + for recipient <- team.owners ++ team.billing_members do + recipient + |> PlausibleWeb.Email.deletion_full_notice_email(team, schedule, summary) + |> Plausible.Mailer.send() + end + + report(schedule, :first_notice, :sent) + + {:error, _} -> + :ok end + else + report(schedule, :first_notice, :cancelled) end end defp send_reminders(today, now) do for schedule <- TeamDeletionSchedules.due_for_reminder(today) do - team = schedule.team + send_reminder(schedule, now) + end + end - if TeamDeletionSchedules.cancel_for_team(team) == :no_schedule do - summary = sites_summary(team) + defp send_reminder(schedule, now) do + team = schedule.team - for recipient <- team.owners ++ team.billing_members do - recipient - |> PlausibleWeb.Email.deletion_reminder_email(team, schedule, summary) - |> Plausible.Mailer.send() - end + if TeamDeletionSchedules.cancel_for_team(team) == :no_schedule do + summary = sites_summary(team) - TeamDeletionSchedules.mark_reminder_sent(schedule, now: now, report_if_invalid?: true) + for recipient <- team.owners ++ team.billing_members do + recipient + |> PlausibleWeb.Email.deletion_reminder_email(team, schedule, summary) + |> Plausible.Mailer.send() end + + TeamDeletionSchedules.mark_reminder_sent(schedule, now: now, report_if_invalid?: true) + report(schedule, :reminder, :sent) + else + report(schedule, :reminder, :cancelled) end end + defp report(schedule, stage, outcome) do + :telemetry.execute(telemetry_run_event(), %{count: 1}, %{ + stage: stage, + outcome: outcome, + category: schedule.category + }) + end + @spec sites_summary(Teams.Team.t()) :: %{domains: [String.t()], more_count: non_neg_integer()} def sites_summary(team) do limit = DeletionSchedule.notification_site_list_limit() diff --git a/lib/workers/unsnooze_team_deletions.ex b/lib/workers/unsnooze_team_deletions.ex index 2d521412619b..9f4c25031d24 100644 --- a/lib/workers/unsnooze_team_deletions.ex +++ b/lib/workers/unsnooze_team_deletions.ex @@ -8,12 +8,19 @@ defmodule Plausible.Workers.UnsnoozeTeamDeletions do alias Plausible.TeamDeletionSchedules + @spec telemetry_run_event() :: [atom()] + def telemetry_run_event(), do: [:plausible, :unsnooze_team_deletions, :run] + @impl Oban.Worker def perform(_job, today \\ Date.utc_today()) do - for schedule <- TeamDeletionSchedules.due_for_unsnooze(today) do + due = TeamDeletionSchedules.due_for_unsnooze(today) + + for schedule <- due do TeamDeletionSchedules.unsnooze(schedule, today: today, report_if_invalid?: true) end + :telemetry.execute(telemetry_run_event(), %{count: length(due)}) + :ok end end diff --git a/test/plausible/team_deletion_schedules_test.exs b/test/plausible/team_deletion_schedules_test.exs index c60553fb015c..4a61b47aac4e 100644 --- a/test/plausible/team_deletion_schedules_test.exs +++ b/test/plausible/team_deletion_schedules_test.exs @@ -37,15 +37,39 @@ defmodule Plausible.TeamDeletionSchedulesTest do assert TeamDeletionSchedules.sync_eligible(@today) == 0 end - test "marks a long-expired trial as backlog, due today" do + test "marks a long-expired trial as backlog, spread across the release window" do team = insert(:team, trial_expiry_date: Date.shift(@today, day: -400)) new_site(team: team) assert TeamDeletionSchedules.sync_eligible(@today) == 1 schedule = Repo.get_by!(TeamDeletionSchedule, team_id: team.id) + window = Plausible.Teams.DeletionSchedule.backlog_release_window_days() + assert schedule.is_backlog - assert schedule.first_notice_due_date == @today + assert schedule.first_notice_due_date == Date.add(@today, rem(team.id, window)) + end + + test "spreads multiple backlog rows across the release window based on team_id" do + teams = + for _ <- 1..5 do + team = insert(:team, trial_expiry_date: Date.shift(@today, day: -400)) + new_site(team: team) + team + end + + assert TeamDeletionSchedules.sync_eligible(@today) == 5 + + window = Plausible.Teams.DeletionSchedule.backlog_release_window_days() + + due_dates = + for team <- teams do + schedule = Repo.get_by!(TeamDeletionSchedule, team_id: team.id) + assert schedule.first_notice_due_date == Date.add(@today, rem(team.id, window)) + schedule.first_notice_due_date + end + + assert length(Enum.uniq(due_dates)) > 1 end test "does not mark a recently-expired trial as backlog" do diff --git a/test/workers/execute_team_deletions_test.exs b/test/workers/execute_team_deletions_test.exs index fb4b22358cf2..0ecb50839b17 100644 --- a/test/workers/execute_team_deletions_test.exs +++ b/test/workers/execute_team_deletions_test.exs @@ -164,5 +164,73 @@ defmodule Plausible.Workers.ExecuteTeamDeletionsTest do assert Repo.reload(site) assert Repo.reload!(schedule).status == :cancelled end + + test "emits telemetry for an executed deletion, tagged by category, with a sites_deleted count", + %{test: test} do + test_pid = self() + telemetry_run = ExecuteTeamDeletions.telemetry_run_event() + telemetry_sites_deleted = ExecuteTeamDeletions.telemetry_sites_deleted_event() + + :telemetry.attach_many( + "#{test}-telemetry-handler", + [telemetry_run, telemetry_sites_deleted], + fn event, measurements, metadata, _ -> + send(test_pid, {:telemetry_handled, event, measurements, metadata}) + end, + %{} + ) + + owner = new_user() + new_site(owner: owner) + team = team_of(owner) |> Plausible.Teams.Team.end_trial() |> Repo.update!() + new_site(team: team) + + insert(:team_deletion_schedule, + team: team, + category: :expired_trial, + status: :reminder_sent, + deletion_date: @today + ) + + assert :ok = ExecuteTeamDeletions.perform(nil, @today) + + assert_receive {:telemetry_handled, ^telemetry_run, %{count: 1}, + %{outcome: :executed, category: :expired_trial}} + + assert_receive {:telemetry_handled, ^telemetry_sites_deleted, %{count: 2}, + %{category: :expired_trial}} + end + + test "emits telemetry for a cancelled schedule, without a sites_deleted event", %{test: test} do + test_pid = self() + telemetry_run = ExecuteTeamDeletions.telemetry_run_event() + + :telemetry.attach( + "#{test}-telemetry-handler", + telemetry_run, + fn event, measurements, metadata, _ -> + send(test_pid, {:telemetry_handled, event, measurements, metadata}) + end, + %{} + ) + + owner = new_user() + new_site(owner: owner) + team = team_of(owner) + + insert(:team_deletion_schedule, + team: team, + category: :churned_subscription, + status: :reminder_sent, + deletion_date: @today + ) + + insert(:subscription, team: team, status: Subscription.Status.active()) + + assert :ok = ExecuteTeamDeletions.perform(nil, @today) + + assert_receive {:telemetry_handled, ^telemetry_run, %{count: 1}, + %{outcome: :cancelled, category: :churned_subscription}} + end end end diff --git a/test/workers/send_deletion_notifications_test.exs b/test/workers/send_deletion_notifications_test.exs index 45a929811ed5..200991952c87 100644 --- a/test/workers/send_deletion_notifications_test.exs +++ b/test/workers/send_deletion_notifications_test.exs @@ -74,6 +74,34 @@ defmodule Plausible.Workers.SendDeletionNotificationsTest do assert updated.deletion_date == Date.shift(@today, day: 30) end + test "sends the backlog first notice email with the finalized deletion date, not the stale placeholder" do + owner = new_user() + new_site(owner: owner) + team = team_of(owner) |> Plausible.Teams.Team.end_trial() |> Repo.update!() + + insert(:team_deletion_schedule, + team: team, + category: :expired_trial, + status: :scheduled, + is_backlog: true, + first_notice_due_date: @today, + deletion_date: ~D[2024-01-01] + ) + + SendDeletionNotifications.perform(nil, @today) + + finalized_date = Date.shift(@today, day: 30) + + assert_email_delivered_with( + to: [{owner.name, owner.email}], + html_body: ~r/#{Regex.escape(PlausibleWeb.EmailView.date_format(finalized_date))}/ + ) + + refute_email_delivered_with( + html_body: ~r/#{Regex.escape(PlausibleWeb.EmailView.date_format(~D[2024-01-01]))}/ + ) + end + test "does not touch a row whose first_notice_due_date hasn't arrived" do owner = new_user() new_site(owner: owner) @@ -189,6 +217,104 @@ defmodule Plausible.Workers.SendDeletionNotificationsTest do end end + describe "telemetry" do + setup %{test: test} do + test_pid = self() + telemetry_run = SendDeletionNotifications.telemetry_run_event() + + :telemetry.attach( + "#{test}-telemetry-handler", + telemetry_run, + fn event, measurements, metadata, _ -> + send(test_pid, {:telemetry_handled, event, measurements, metadata}) + end, + %{} + ) + + %{telemetry_run: telemetry_run} + end + + test "emits a :sent first notice event", %{telemetry_run: telemetry_run} do + owner = new_user() + new_site(owner: owner) + team = team_of(owner) |> Plausible.Teams.Team.end_trial() |> Repo.update!() + + insert(:team_deletion_schedule, + team: team, + category: :churned_subscription, + status: :scheduled, + first_notice_due_date: @today + ) + + SendDeletionNotifications.perform(nil, @today) + + assert_receive {:telemetry_handled, ^telemetry_run, %{count: 1}, + %{stage: :first_notice, outcome: :sent, category: :churned_subscription}} + end + + test "emits a :cancelled first notice event when the team has reactivated", %{ + telemetry_run: telemetry_run + } do + owner = new_user() + new_site(owner: owner) + team = team_of(owner) + + insert(:team_deletion_schedule, + team: team, + category: :expired_trial, + status: :scheduled, + first_notice_due_date: @today + ) + + insert(:subscription, team: team, status: Subscription.Status.active()) + + SendDeletionNotifications.perform(nil, @today) + + assert_receive {:telemetry_handled, ^telemetry_run, %{count: 1}, + %{stage: :first_notice, outcome: :cancelled, category: :expired_trial}} + end + + test "emits a :sent reminder event", %{telemetry_run: telemetry_run} do + owner = new_user() + new_site(owner: owner) + team = team_of(owner) |> Plausible.Teams.Team.end_trial() |> Repo.update!() + + insert(:team_deletion_schedule, + team: team, + category: :churned_subscription, + status: :first_notice_sent, + deletion_date: Date.shift(@today, day: 3) + ) + + SendDeletionNotifications.perform(nil, @today) + + assert_receive {:telemetry_handled, ^telemetry_run, %{count: 1}, + %{stage: :reminder, outcome: :sent, category: :churned_subscription}} + end + + test "emits a :cancelled reminder event when the team has reactivated", %{ + telemetry_run: telemetry_run + } do + owner = new_user() + new_site(owner: owner) + team = team_of(owner) + + insert(:team_deletion_schedule, + team: team, + category: :expired_trial, + status: :first_notice_sent, + deletion_date: Date.shift(@today, day: 3) + ) + + insert(:subscription, team: team, status: Subscription.Status.active()) + + SendDeletionNotifications.perform(nil, @today) + + assert_receive {:telemetry_handled, ^telemetry_run, %{count: 1}, + %{stage: :reminder, outcome: :cancelled, category: :expired_trial}} + end + end + describe "sites_summary/1" do test "returns every domain uncapped when the team has few sites" do owner = new_user() diff --git a/test/workers/unsnooze_team_deletions_test.exs b/test/workers/unsnooze_team_deletions_test.exs index 845814794fcb..1ba2ab4c19a2 100644 --- a/test/workers/unsnooze_team_deletions_test.exs +++ b/test/workers/unsnooze_team_deletions_test.exs @@ -62,5 +62,26 @@ defmodule Plausible.Workers.UnsnoozeTeamDeletionsTest do assert Repo.reload!(schedule).status == :reminder_sent end + + test "emits telemetry with the number of unsnoozed schedules", %{test: test} do + test_pid = self() + telemetry_run = UnsnoozeTeamDeletions.telemetry_run_event() + + :telemetry.attach( + "#{test}-telemetry-handler", + telemetry_run, + fn event, measurements, metadata, _ -> + send(test_pid, {:telemetry_handled, event, measurements, metadata}) + end, + %{} + ) + + insert(:team_deletion_schedule, status: :snoozed, snoozed_until: @today) + insert(:team_deletion_schedule, status: :snoozed, snoozed_until: Date.shift(@today, day: 1)) + + assert :ok = UnsnoozeTeamDeletions.perform(nil, @today) + + assert_receive {:telemetry_handled, ^telemetry_run, %{count: 1}, %{}} + end end end