Commit d0594889 authored by Mikko Ahlroth's avatar Mikko Ahlroth

Rework XP per user caching system

CachedXP is removed in favor of a JSON field on the user model, which can
be used to add all kinds of different caches. Caching is implemented in
this field for languages, machines and dates. This removes the need to hit
the database for other uses than to update the cache and to load the last
12 hours' XP data. As a result the profile view's display should be much
faster.
parent 23dbde8a
......@@ -16,20 +16,26 @@ defmodule CodeStats.XPCacheRefresher do
end
def init(state) do
do_refresh()
Process.send_after(self(), :work, @how_often)
{:ok, state}
end
def handle_info(:work, state) do
(from u in User, select: u)
|> Repo.all()
|> Enum.each(
fn user -> User.update_cached_xps(user, true) end
)
do_refresh()
# Start the timer again
Process.send_after(self(), :work, @how_often)
{:noreply, state}
end
defp do_refresh() do
(from u in User, select: u)
|> Repo.all()
|> Enum.each(
fn user -> User.update_cached_xps(user, true) end
)
end
end
......@@ -3,7 +3,7 @@ defmodule CodeStats.Mixfile do
def project do
[app: :code_stats,
version: "1.4.0",
version: "1.5.0",
elixir: "~> 1.2",
elixirc_paths: elixirc_paths(Mix.env),
compilers: [:phoenix, :gettext] ++ Mix.compilers,
......
defmodule CodeStats.Repo.Migrations.AddUserCache do
use Ecto.Migration
def change do
alter table(:users) do
add :cache, :jsonb
end
end
end
defmodule CodeStats.Repo.Migrations.RemoveCachedXps do
use Ecto.Migration
def change do
drop table(:cached_xps)
end
end
......@@ -3,15 +3,17 @@ defmodule CodeStats.ProfileController do
import Ecto.Query, only: [from: 2]
alias CodeStats.Repo
alias CodeStats.{AuthUtils, PermissionUtils}
alias CodeStats.User
alias CodeStats.SetSessionUser
alias CodeStats.Pulse
alias CodeStats.XP
alias CodeStats.Language
alias CodeStats.Machine
alias CodeStats.{
Repo,
AuthUtils,
PermissionUtils,
User,
SetSessionUser,
Pulse,
XP,
Language,
Machine
}
def my_profile(conn, _params) do
user = SetSessionUser.get_user_data(conn)
......@@ -39,24 +41,37 @@ defmodule CodeStats.ProfileController do
end
def render_profile(conn, user) do
now = Calendar.DateTime.now_utc()
# Update and get user's cache data
%{
languages: language_xps,
machines: machine_xps,
dates: date_xps
} = User.update_cached_xps(user)
# Calculate total XP
total_xp = Map.to_list(language_xps)
|> Enum.reduce(0, fn {_, amount}, acc -> acc + amount end)
# Fetch necessary language and machine objects for cached data and sort them
language_xps = process_language_xps(language_xps)
machine_xps = process_machine_xps(machine_xps, user)
date_xps = process_date_xps(date_xps)
# Get new XP data from last 12 hours
now = DateTime.utc_now()
latest_xp_since = Calendar.DateTime.subtract!(now, 3600 * 12)
xps = User.update_cached_xps(user)
|> Enum.sort(fn a, b -> a.amount >= b.amount end)
new_xps = get_latest_xps(user, latest_xp_since)
{machine_xps, new_machine_xps} = get_machine_xps(user, latest_xp_since)
days_coded = get_days_coded(user)
total_xp = Enum.reduce(xps, 0, fn xp, acc -> acc + xp.amount end)
new_machine_xps = get_machine_xps(user, latest_xp_since)
total_new_xp = Enum.reduce(Map.values(new_xps), 0, fn amount, acc -> acc + amount end)
{highlighted_xps, more_xps} = Enum.split(xps, 10)
last_day_coded = case Enum.empty?(date_xps) do
true -> nil
_ -> date_xps |> Enum.at(0) |> elem(0)
end
last_day_coded = Enum.at(days_coded, 0)
xp_per_day = case last_day_coded do
nil -> 0
_ -> trunc(Float.round(total_xp / Enum.count(days_coded)))
_ -> trunc(Float.round(total_xp / Enum.count(date_xps)))
end
conn
......@@ -65,9 +80,8 @@ defmodule CodeStats.ProfileController do
|> assign(:total_xp, total_xp)
|> assign(:last_day_coded, last_day_coded)
|> assign(:xp_per_day, xp_per_day)
|> assign(:xps, highlighted_xps)
|> assign(:more_xps, more_xps)
|> assign(:new_xps, new_xps)
|> assign(:language_xps, language_xps)
|> assign(:machine_xps, machine_xps)
|> assign(:new_machine_xps, new_machine_xps)
|> assign(:total_new_xp, total_new_xp)
......@@ -94,19 +108,6 @@ defmodule CodeStats.ProfileController do
# Get all XP per machine and XP per machine per last 12 hours
defp get_machine_xps(user, then) do
xps_q = from m in Machine,
join: p in Pulse, on: m.id == p.machine_id,
join: x in XP, on: p.id == x.pulse_id,
where: m.user_id == ^user.id,
group_by: m.id,
order_by: [desc: sum(x.amount)],
select: {m, sum(x.amount)}
xps = case Repo.all(xps_q) do
nil -> []
ret -> ret
end
new_xps_q = from m in Machine,
join: p in Pulse, on: m.id == p.machine_id,
join: x in XP, on: p.id == x.pulse_id,
......@@ -115,29 +116,51 @@ defmodule CodeStats.ProfileController do
order_by: [desc: sum(x.amount)],
select: {m, sum(x.amount)}
new_xps = case Repo.all(new_xps_q) do
case Repo.all(new_xps_q) do
nil -> %{}
ret ->
Enum.reduce(ret, %{}, fn {machine, amount}, acc ->
Map.put(acc, machine.id, amount)
end)
end
end
defp process_language_xps(language_xps) do
language_xps = Map.to_list(language_xps)
|> Enum.sort(fn {_, a}, {_, b} -> a > b end)
language_ids = Enum.map(language_xps, fn {id, _} -> id end)
{xps, new_xps}
language_q = from l in Language,
where: l.id in ^language_ids,
select: {l.id, l}
languages = Repo.all(language_q) |> Map.new()
Enum.map(language_xps, fn {id, amount} ->
{Map.get(languages, id), amount}
end)
end
# Get amount of days when user has coded at least something
defp get_days_coded(user) do
days_q = from p in Pulse,
where: p.user_id == ^user.id,
group_by: fragment("DATE(?)", p.sent_at),
select: fragment("DATE(?)", p.sent_at),
order_by: [desc: fragment("DATE(?)", p.sent_at)]
case Repo.all(days_q) do
nil -> []
ret -> ret
end
defp process_machine_xps(machine_xps, user) do
machine_xps = Map.to_list(machine_xps)
|> Enum.sort(fn {_, a}, {_, b} -> a > b end)
machine_q = from m in Machine,
where: m.user_id == ^user.id,
select: {m.id, m}
machines = Repo.all(machine_q) |> Map.new()
Enum.map(machine_xps, fn {id, amount} ->
{Map.get(machines, id), amount}
end)
end
defp process_date_xps(date_xps) do
date_xps
|> Map.to_list()
|> Enum.sort(fn {_, a}, {_, b} -> a > b end)
end
# Fix the username specified in the URL by converting plus characters to spaces.
......
defmodule CodeStats.CachedXP do
@moduledoc """
Cached XP is precalculated XP for a certain user in a certain language.
A user should only have one CachedXP per language. These exist to lighten the load on
the database, so that not all pulses need to be loaded on every request.
"""
use CodeStats.Web, :model
schema "cached_xps" do
field :amount, :integer
belongs_to :user, CodeStats.User
belongs_to :language, CodeStats.Language
timestamps
end
@required_fields ~w(amount)
@optional_fields ~w()
@doc """
Creates a changeset based on the `model` and `params`.
If no params are provided, an invalid changeset is returned
with no validation performed.
"""
def changeset(model, params \\ %{}) do
model
|> cast(params, @required_fields, @optional_fields)
end
end
......@@ -8,11 +8,11 @@ defmodule CodeStats.User do
import Ecto.Query, only: [from: 2]
alias CodeStats.Repo
alias CodeStats.Pulse
alias CodeStats.Language
alias CodeStats.XP
alias CodeStats.CachedXP
alias CodeStats.{
Repo,
Pulse,
XP
}
schema "users" do
field :username, :string
......@@ -20,9 +20,9 @@ defmodule CodeStats.User do
field :password, :string
field :last_cached, Calecto.DateTimeUTC
field :private_profile, :boolean
field :cache, :map
has_many :pulses, Pulse
has_many :cached_xps, CachedXP
timestamps
end
......@@ -73,13 +73,10 @@ defmodule CodeStats.User do
end
@doc """
Calculate and store CachedXP values for user.
Calculate and store cached XP values for user.
Will first load all existing CachedXP, then sum any new XPs per Language and add those to
the CachedXPs, creating new ones if required.
If `update_all` is set, all XP is gathered and CachedXP is replaced, not just added to. This
results in a total recalculation of all the user's XP.
If `update_all` is set, all XP is gathered and the whole cache is replaced, not
just added to. This results in a total recalculation of all the user's XP.
"""
def update_cached_xps(user, update_all \\ false) do
last_cached = if not update_all and user.last_cached != nil do
......@@ -89,83 +86,119 @@ defmodule CodeStats.User do
datetime
end
cached_xps_q = from cx in CachedXP,
where: cx.user_id == ^user.id,
preload: [:language]
cached_xps = case Repo.all(cached_xps_q) do
nil -> []
ret -> ret
# If update_all is given, don't use any previous cache data
cached_data = case update_all do
false -> unformat_cache_from_db(user.cache)
true -> %{
languages: %{},
machines: %{},
dates: %{}
}
end
|> Enum.reduce(%{}, fn cached_xp, acc ->
# Each cached XP is inserted into a 3-tuple of {CachedXP, dirty bit, amount}
# The dirty bit is used to persist only changed CachedXPs
# If we are updating all XP, every CachedXP will be marked as dirty and will have
# amount set to 0 to start with
if update_all do
Map.put(acc, cached_xp.language_id, {cached_xp, true, 0})
else
Map.put(acc, cached_xp.language_id, {cached_xp, false, cached_xp.amount})
end
end)
# Load all of user's new XP plus the Language for each XP
# Load all of user's new XP plus required associations
xps_q = from x in XP,
join: p in Pulse, on: p.id == x.pulse_id,
join: l in Language, on: l.id == x.language_id,
where: p.user_id == ^user.id and p.inserted_at >= ^last_cached,
select: {x, l}
select: {p, x}
xps = case Repo.all(xps_q) do
nil -> []
ret -> ret
end
# Reduce over all new xps
updated_cached_xps = Enum.reduce(xps, cached_xps, fn {xp, language}, cached_xps ->
{cached_xp, _, amount} = Map.get(
cached_xps,
xp.language_id,
{
%CachedXP{
language: language,
language_id: language.id,
user_id: user.id,
amount: 0
},
true,
0
}
)
Map.put(cached_xps, xp.language_id, {cached_xp, true, amount + xp.amount})
end)
|> Map.values()
# Persist changed (dirty) cached XPs
updated_cached_xps
|> Enum.filter(fn {_, dirty, _} -> dirty end)
|> Enum.each(fn {cached_xp, _, amount} ->
CachedXP.changeset(cached_xp, %{"amount" => amount})
|> Changeset.put_change(:language_id, cached_xp.language_id)
|> Changeset.put_change(:user_id, user.id)
|> Repo.insert_or_update!()
end)
language_data = generate_language_cache(cached_data.languages, xps)
machine_data = generate_machine_cache(cached_data.machines, xps)
date_data = generate_date_cache(cached_data.dates, xps)
final_cache = %{
languages: language_data,
machines: machine_data,
dates: date_data
}
# Persist cache changes
user
|> cast(%{cache: format_cache_for_db(final_cache)}, [:cache])
|> Repo.update!()
# Finally update the user's last_cached timestamp
updating_changeset(user, %{})
|> Changeset.put_change(:last_cached, Calendar.DateTime.now_utc())
|> Repo.update()
|> Repo.update!()
# Return the cache data for the caller
final_cache
end
defp generate_language_cache(language_data, xps) do
Enum.reduce(xps, language_data, fn {_, xp}, acc ->
Map.get_and_update(acc, xp.language_id, fn old_val ->
{old_val, val_or_0(old_val) + xp.amount}
end)
|> elem(1)
end)
end
defp generate_machine_cache(machine_data, xps) do
Enum.reduce(xps, machine_data, fn {pulse, xp}, acc ->
Map.get_and_update(acc, pulse.machine_id, fn old_val ->
{old_val, val_or_0(old_val) + xp.amount}
end)
|> elem(1)
end)
end
# Return all of the user's cached XPs
updated_cached_xps
|> Enum.map(fn
{cached_xp, false, _} -> cached_xp
{cached_xp, true, amount} -> %{cached_xp | amount: amount}
defp generate_date_cache(date_data, xps) do
Enum.reduce(xps, date_data, fn {pulse, xp}, acc ->
date = DateTime.to_date(pulse.sent_at)
Map.get_and_update(acc, date, fn old_val ->
{old_val, val_or_0(old_val) + xp.amount}
end)
|> elem(1)
end)
end
# Format data in cache for storing into db as JSON
defp format_cache_for_db(cache) do
languages = Map.get(cache, :languages)
|> int_keys_to_str()
machines = Map.get(cache, :machines)
|> int_keys_to_str()
dates = Map.get(cache, :dates)
|> Map.to_list()
|> Enum.map(fn {key, value} -> {Date.to_iso8601(key), value} end)
|> Map.new()
%{
languages: languages,
machines: machines,
dates: dates
}
end
# Unformat data from DB to native datatypes
defp unformat_cache_from_db(cache) do
languages = Map.get(cache, "languages")
|> str_keys_to_int()
machines = Map.get(cache, "machines")
|> str_keys_to_int()
dates = Map.get(cache, "dates")
|> Map.to_list()
|> Enum.map(fn {key, value} -> {Date.from_iso8601!(key), value} end)
|> Map.new()
%{
languages: languages,
machines: machines,
dates: dates
}
end
defp hash_password(password) do
Bcrypt.hashpwsalt(password)
end
......@@ -174,4 +207,21 @@ defmodule CodeStats.User do
changeset
|> validate_format(:email, ~r/^$|@/)
end
defp val_or_0(nil), do: 0
defp val_or_0(val) when is_number(val), do: val
defp int_keys_to_str(map) do
map
|> Map.to_list()
|> Enum.map(fn {key, value} -> {Integer.to_string(key), value} end)
|> Map.new()
end
defp str_keys_to_int(map) do
map
|> Map.to_list()
|> Enum.map(fn {key, value} -> {Integer.parse(key) |> elem(0), value} end)
|> Map.new()
end
end
......@@ -29,6 +29,13 @@
<h2>Changelog</h2>
<h3>1.5.0 – 2016-07-17 – Second performance update</h3>
<ul>
<li>Earlier, profile view XPs per language were cached to the database as CachedXP elements. This worked fine for that purpose but caching for other dimensions such as per machine and per date were needed. In this update, CachedXP is removed in favor of a JSON field on the user model, which can be used to add all kinds of different caches. Caching is implemented in this field for languages, machines and dates. This removes the need to hit the database for other uses than to update the cache and to load the last 12 hours' XP data. As a result the profile view's display should be much faster.</li>
<li>An issue with 1.4.0 and database limits in queries was hotfixed.</li>
</ul>
<h3>1.4.0 – 2016-07-16 – First performance update</h3>
<ul>
......
......@@ -54,7 +54,7 @@
</div>
</div>
<%= if Enum.empty?(@xps) do %>
<%= if not has_language_xps?(@language_xps) do %>
<div class="jumbotron">
<p class="lead">
<%= if get_current_user_id(@conn) == @user.id do %>
......@@ -69,15 +69,15 @@
</div>
<% else %>
<div class="row">
<%= for xp <- @xps do %>
<% new_xp = Map.get(@new_xps, xp.language.name, 0) %>
<%= for {language, xp} <- (split_language_xps(@language_xps) |> elem(0)) do %>
<% new_xp = Map.get(@new_xps, language.name, 0) %>
<div class="col-xs-12 language-progress">
<h4>
<%= xp.language.name %>
<%= language.name %>
level
<%= get_level(xp.amount) %>
(<%= format_xp(xp.amount) %>&nbsp;XP)
<%= get_level(xp) %>
(<%= format_xp(xp) %>&nbsp;XP)
<%= if new_xp > 0 do %>
<sup>
......@@ -86,7 +86,7 @@
<% end %>
</h4>
<div class="progress">
<% {old_width, new_width} = get_xp_bar_widths(xp.amount, new_xp) %>
<% {old_width, new_width} = get_xp_bar_widths(xp, new_xp) %>
<div class="progress-bar progress-bar-success" role="progressbar" style="width: <%= old_width %>%" aria-valuenow="<%= old_width %>" aria-valuemin="0" aria-valuemax="100">
<span class="sr-only">Level progress <%= old_width %> %.</span>
</div>
......@@ -120,18 +120,18 @@
</div>
<div class="row">
<%= if not Enum.empty?(@more_xps) do %>
<%= if has_more_language_xps?(@language_xps) do %>
<div class="col-xs-12 col-sm-6">
<h4>Other languages</h4>
<ol start="11">
<%= for xp <- @more_xps do %>
<% new_xp = Map.get(@new_xps, xp.language.name, 0) %>
<%= for {language, xp} <- (split_language_xps(@language_xps) |> elem(1)) do %>
<% new_xp = Map.get(@new_xps, language.name, 0) %>
<li>
<strong><%= xp.language.name %></strong>
<strong><%= language.name %></strong>
level
<%= get_level(xp.amount) %>
(<%= format_xp(xp.amount) %>&nbsp;XP)
<%= get_level(xp) %>
(<%= format_xp(xp) %>&nbsp;XP)
<%= if new_xp > 0 do %>
<sup>
......@@ -144,7 +144,7 @@
</div>
<% end %>
<%= if not Enum.empty?(@machine_xps) do %>
<%= if has_machine_xps?(@machine_xps) do %>
<div class="col-xs-12 col-sm-6">
<h4>Machines</h4>
......
defmodule CodeStats.ProfileView do
use CodeStats.Web, :view
# How many language XPs to display with progress bars
@language_xp_amount 10
def get_xp_bar_widths(total_xp, new_xp) do
level = get_level(total_xp)
current_level_xp = get_next_level_xp(level - 1)
......@@ -19,4 +22,20 @@ defmodule CodeStats.ProfileView do
}
end
end
def has_language_xps?(language_xps) do
not Enum.empty?(language_xps)
end
def split_language_xps(language_xps) do
Enum.split(language_xps, @language_xp_amount)
end
def has_more_language_xps?(language_xps) do
Enum.count(language_xps) > @language_xp_amount
end
def has_machine_xps?(machine_xps) do
not Enum.empty?(machine_xps)
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment