Commit 7b20d530 authored by Mikko Ahlroth's avatar Mikko Ahlroth

Redo the caching stuff to remove useless 24 hrs cache

parent 986017e6
Pipeline #39525212 passed with stages
in 8 minutes and 36 seconds
......@@ -103,7 +103,7 @@ defmodule CodeStats.User do
case Repo.update(cset) do
{:ok, _} -> :ok
{:error, %Ecto.Changeset{} = err_cset} -> err_cset.errors
{:error, %Ecto.Changeset{} = err_cset} -> {:error, err_cset.errors}
......@@ -134,11 +134,11 @@ defmodule CodeStats.User do
@doc """
Calculate and store cached XP values for user.
If `update_all` is set, XP is gathered since the given datetime. If the value is `:all`, then
all of the users XP is processed.
If `update_all` is set, all of the user's cache is regenerated. Otherwise the cache is added to
with the data after the last caching time.
@spec update_cached_xps(%__MODULE__{}, nil | :all | DateTime.t()) :: map
def update_cached_xps(user, since \\ nil) do
@spec update_cached_xps(%__MODULE__{}, boolean) :: map
def update_cached_xps(user, update_all \\ false) do
update_start_time = DateTime.utc_now()
# If update_all is given or user cache is empty, don't use any previous cache data
......@@ -156,15 +156,10 @@ defmodule CodeStats.User do
all_since = DateTime.from_naive!(~N[1970-01-01T00:00:00], "Etc/UTC")
{xp_since, cached_data} =
cond do
match?(%DateTime{}, since) ->
{since, empty_cache}
since == :all or is_nil(user.last_cached) ->
{all_since, unformat_cache_from_db(user.cache)}
true ->
{user.last_cached, unformat_cache_from_db(user.cache)}
if update_all or is_nil(user.last_cached) do
{all_since, empty_cache}
{user.last_cached, unformat_cache_from_db(user.cache)}
# Load all of user's new XP plus required associations
......@@ -196,7 +191,7 @@ defmodule CodeStats.User do
# Correct key for storing caching duration
duration_key = if since == :all, do: :total_caching_duration, else: :caching_duration
duration_key = if update_all, do: :total_caching_duration, else: :caching_duration
# Store cache that is formatted for DB and add caching duration
stored_cache =
......@@ -12,12 +12,6 @@ defmodule CodeStats.XP.XPCacheRefresher do
# Run about every second minute (120 seconds after last run)
@how_often 2 * 60 * 1000
# How many seconds back to fetch data for the "last 24h sync"
@sync_24h_secs 24 * 60 * 60
# How many users to sync for the "last 24h sync"
@sync_24h_count 50
# How many users to sync totally
@sync_total_count 1
......@@ -42,40 +36,13 @@ defmodule CodeStats.XP.XPCacheRefresher do
@doc """
Refresh XP caches users in the system.
Will pick a bunch of the users that have not been synced in the last 24 hours (least recently
synced first), and sync their data from the last 24 hours. For this sync, if the user does not
have any pulses added after the last cache time, they are not synced.
After that, pick a smaller list of users (least recently synced first) and sync them totally.
Refresh XP caches of a number of users in the system. The least recently cached users are picked
@spec do_refresh() :: :ok
def do_refresh() do
defp sync_24h() do
now = DateTime.utc_now()
then = Calendar.DateTime.subtract!(now, @sync_24h_secs)
from(u in User,
join: p in User.Pulse,
on: p.user_id ==,
where: u.last_cached < ^then and p.inserted_at > ^then,
having: count(p) > 0,
order_by: [asc: u.last_cached],
limit: @sync_24h_count
|> Repo.all()
|> Enum.each(&User.update_cached_xps(&1, then))
defp sync_total() do
from(u in User, order_by: [asc: u.last_cached], limit: @sync_total_count)
|> Repo.all()
|> Enum.each(&User.update_cached_xps(&1, :all))
|> Enum.each(&User.update_cached_xps(&1, true))
......@@ -34,7 +34,7 @@ defmodule CodeStatsWeb.AuthController do
# If ret is nil, user was not found -> run dummy auth to prevent user enumeration
# But they can enumerate with the signin form anyway lol
# TODO: Add CAPTCHA to signup form
if ret == nil, do: AuthUtils.dummy_auth_user()
if is_nil(ret), do: AuthUtils.dummy_auth_user()
|> assign(:title, "Login")
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment