Commit 975dd7ed authored by Mikko Ahlroth's avatar Mikko Ahlroth

Added CacheService for total XP caching

This is a major performance boost for the front page. All total XP per
language is saved in ETS where it can be served from very quickly.

Also:
* Fixed last coded at dates in profile view
* Fixed some JSON API errors not showing up
parent 8603d76a
......@@ -16,6 +16,9 @@ defmodule CodeStats do
# Start XPCacheRefresher
worker(CodeStats.XPCacheRefresher, []),
# Start cache service
worker(CodeStats.CacheService, []),
]
# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
......
defmodule CodeStats.CacheService do
@moduledoc """
The cache service implements a simple key-value cache that stores its values in ETS
(Erlang Term Storage) and optionally refreshes them periodically.
It also manages ownership of the ETS tables of the cache. The tables are set to public so that
outside processes (such as request handlers) can add values to the cache.
"""
use GenServer
import Ecto.Query, only: [from: 2]
alias CodeStats.{Language, XP, Repo}
# Table names
@language_xp_cache_table :cache_service_language_xp_cache
# Timers for refreshing data
@total_language_xp_refresh_timer 900 * 1000
def start_link() do
GenServer.start_link(__MODULE__, %{})
end
def init(state) do
:ets.new(@language_xp_cache_table, [:named_table, :set, :public, read_concurrency: true, write_concurrency: true])
refresh_total_language_xp_and_repeat()
{:ok, state}
end
@doc """
Add the given amount of XP for the given language in the total language XP cache.
Returns the new amount of XP for the language in the cache.
"""
@spec add_total_language_xp(%Language{}, integer) :: integer
def add_total_language_xp(language, value) do
# Form key for possible future use
key = {language.name, :total}
case :ets.update_counter(@language_xp_cache_table, key, {2, value}, {key, value}) do
new_count when is_integer(new_count) -> new_count
_ -> raise "Updating counter #{inspect key} failed!"
end
end
@doc """
Get the total XP in the system for each language. Returns a list of tuples where the first
element is the language name and the second element is the amount of XP.
"""
@spec get_total_language_xps() :: [{String.t, integer}]
def get_total_language_xps() do
:ets.match_object(@language_xp_cache_table, {{:"$1", :total}, :"$2"})
|> Enum.map(fn {{lang_name, :total}, amount} -> {lang_name, amount} end)
end
def handle_info(:refresh_total_language_xp, state) do
refresh_total_language_xp_and_repeat()
{:noreply, state}
end
defp refresh_total_language_xp_and_repeat() do
refresh_total_language_xp()
Process.send_after(self(), :refresh_total_language_xp, @total_language_xp_refresh_timer)
end
defp refresh_total_language_xp() do
# Refresh all data in total language XP cache
most_popular_q = from x in XP,
join: l in Language, on: l.id == x.language_id,
group_by: l.id,
order_by: [desc: sum(x.amount)],
select: {l, sum(x.amount)},
limit: 10
most_popular = case Repo.all(most_popular_q) do
nil -> []
ret -> ret
end
|> Enum.map(fn {lang, amount} -> {{lang.name, :total}, amount} end)
:ets.insert(@language_xp_cache_table, most_popular)
end
end
......@@ -3,7 +3,7 @@ defmodule CodeStats.Mixfile do
def project do
[app: :code_stats,
version: "1.3.7",
version: "1.4.0",
elixir: "~> 1.2",
elixirc_paths: elixirc_paths(Mix.env),
compilers: [:phoenix, :gettext] ++ Mix.compilers,
......
......@@ -12,14 +12,14 @@
"ex_doc": {:hex, :ex_doc, "0.12.0", "b774aabfede4af31c0301aece12371cbd25995a21bb3d71d66f5c2fe074c603f", [:mix], [{:earmark, "~> 0.2", [hex: :earmark, optional: false]}]},
"fs": {:hex, :fs, "0.9.2", "ed17036c26c3f70ac49781ed9220a50c36775c6ca2cf8182d123b6566e49ec59", [:rebar], []},
"gettext": {:hex, :gettext, "0.11.0", "80c1dd42d270482418fa158ec5ba073d2980e3718bacad86f3d4ad71d5667679", [:mix], []},
"hackney": {:hex, :hackney, "1.6.0", "8d1e9440c9edf23bf5e5e2fe0c71de03eb265103b72901337394c840eec679ac", [:rebar3], [{:ssl_verify_fun, "1.1.0", [hex: :ssl_verify_fun, optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, optional: false]}, {:metrics, "1.0.1", [hex: :metrics, optional: false]}, {:idna, "1.2.0", [hex: :idna, optional: false]}, {:certifi, "0.4.0", [hex: :certifi, optional: false]}]},
"hackney": {:hex, :hackney, "1.6.1", "ddd22d42db2b50e6a155439c8811b8f6df61a4395de10509714ad2751c6da817", [:rebar3], [{:ssl_verify_fun, "1.1.0", [hex: :ssl_verify_fun, optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, optional: false]}, {:metrics, "1.0.1", [hex: :metrics, optional: false]}, {:idna, "1.2.0", [hex: :idna, optional: false]}, {:certifi, "0.4.0", [hex: :certifi, optional: false]}]},
"idna": {:hex, :idna, "1.2.0", "ac62ee99da068f43c50dc69acf700e03a62a348360126260e87f2b54eced86b2", [:rebar3], []},
"metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], []},
"mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], []},
"number": {:hex, :number, "0.4.2", "027fc6b03d17d3ccd4beea599c35503ed81901284749048ea2f4519ec877727a", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, optional: true]}]},
"phoenix": {:hex, :phoenix, "1.2.0", "1bdeb99c254f4c534cdf98fd201dede682297ccc62fcac5d57a2627c3b6681fb", [:mix], [{:poison, "~> 1.5 or ~> 2.0", [hex: :poison, optional: false]}, {:phoenix_pubsub, "~> 1.0", [hex: :phoenix_pubsub, optional: false]}, {:plug, "~> 1.1", [hex: :plug, optional: false]}, {:cowboy, "~> 1.0", [hex: :cowboy, optional: true]}]},
"phoenix_ecto": {:hex, :phoenix_ecto, "3.0.0", "b947aaf03d076f5b1448f87828f22fb7710478ee38455c67cc3fe8e9a4dfd015", [:mix], [{:ecto, "~> 2.0.0-rc", [hex: :ecto, optional: false]}, {:phoenix_html, "~> 2.6", [hex: :phoenix_html, optional: true]}]},
"phoenix_html": {:hex, :phoenix_html, "2.6.0", "b9f7e091eb3d908586d9634596478fb9e577ee033d76f4ff327a745569bdd2d8", [:mix], [{:plug, "~> 0.13 or ~> 1.0", [hex: :plug, optional: false]}]},
"phoenix_html": {:hex, :phoenix_html, "2.6.2", "944a5e581b0d899e4f4c838a69503ebd05300fe35ba228a74439e6253e10e0c0", [:mix], [{:plug, "~> 1.0", [hex: :plug, optional: false]}]},
"phoenix_live_reload": {:hex, :phoenix_live_reload, "1.0.5", "829218c4152ba1e9848e2bf8e161fcde6b4ec679a516259442561d21fde68d0b", [:mix], [{:phoenix, "~> 1.0 or ~> 1.2-rc", [hex: :phoenix, optional: false]}, {:fs, "~> 0.9.1", [hex: :fs, optional: false]}]},
"phoenix_pubsub": {:hex, :phoenix_pubsub, "1.0.0", "c31af4be22afeeebfaf246592778c8c840e5a1ddc7ca87610c41ccfb160c2c57", [:mix], []},
"plug": {:hex, :plug, "1.1.6", "8927e4028433fcb859e000b9389ee9c37c80eb28378eeeea31b0273350bf668b", [:mix], [{:cowboy, "~> 1.0", [hex: :cowboy, optional: true]}]},
......
......@@ -3,22 +3,28 @@ defmodule CodeStats.PageController do
import Ecto.Query, only: [from: 2]
alias CodeStats.Repo
alias CodeStats.XP
alias CodeStats.Pulse
alias CodeStats.Language
alias CodeStats.{
Repo,
XP,
Pulse,
Language,
CacheService
}
@popular_languages_limit 10
def index(conn, _params) do
now = Calendar.DateTime.now_utc()
then = Calendar.DateTime.subtract!(now, 3600 * 12)
total_xp_q = from x in XP,
select: sum(x.amount)
# Load total language XPs from cache and use them to populate total XP and
# list of most popular languages
total_lang_xps = CacheService.get_total_language_xps()
|> Enum.sort(fn {_, a}, {_, b} -> a > b end)
total_xp = case Repo.one(total_xp_q) do
nil -> 0
ret -> ret
end
total_xp = Enum.reduce(total_lang_xps, 0, fn {_, amount}, acc -> amount + acc end)
most_popular = Enum.slice(total_lang_xps, 0..@popular_languages_limit)
last_12h_xp_q = from x in XP,
join: p in Pulse, on: p.id == x.pulse_id,
......@@ -30,18 +36,6 @@ defmodule CodeStats.PageController do
ret -> ret
end
most_popular_q = from x in XP,
join: l in Language, on: l.id == x.language_id,
group_by: l.id,
order_by: [desc: sum(x.amount)],
select: {l.name, sum(x.amount)},
limit: 10
most_popular = case Repo.all(most_popular_q) do
nil -> []
ret -> ret
end
most_popular_12h_q = from x in XP,
join: l in Language, on: l.id == x.language_id,
join: p in Pulse, on: p.id == x.pulse_id,
......@@ -49,7 +43,7 @@ defmodule CodeStats.PageController do
group_by: l.id,
order_by: [desc: sum(x.amount)],
select: {l.name, sum(x.amount)},
limit: 10
limit: @popular_languages_limit
most_popular_12h = case Repo.all(most_popular_12h_q) do
nil -> []
......
......@@ -8,44 +8,40 @@ defmodule CodeStats.PulseController do
alias Calendar.DateTime, as: CDateTime
alias CodeStats.AuthUtils
alias CodeStats.Repo
alias CodeStats.Pulse
alias CodeStats.Language
alias CodeStats.XP
alias CodeStats.{
AuthUtils,
Repo,
Pulse,
Language,
XP,
CacheService
}
def add(conn, %{"coded_at" => timestamp, "xps" => xps}) do
if not is_list(xps) do
resp(conn, 400, %{error: "Invalid xps format."})
end
case do_add(conn, timestamp, xps) do
:ok ->
conn
|> put_status(201)
|> json(%{"ok" => "Great success!"})
{:error, :not_found, reason} ->
resp(conn, 404, %{error: reason})
else
{:error, :generic, reason} ->
resp(conn, 400, %{error: reason})
{user, machine} = AuthUtils.get_api_details(conn)
{:error, :internal, reason} ->
resp(conn, 500, %{error: reason})
end
end
with {:ok, %DateTime{} = datetime} <- parse_timestamp(timestamp),
{:ok, datetime} <- check_datetime_diff(datetime),
{:ok, %Pulse{} = pulse} <- create_pulse(user, machine, datetime),
{:ok, inserted_xps} <- create_xps(pulse, xps),
:ok <- update_caches(inserted_xps)
do
conn |> put_status(201) |> json(%{"ok" => "Great success!"})
else
{:error, :not_found, reason} ->
conn |> put_status(404) |> json(%{"error" => reason})
defp do_add(conn, timestamp, xps) do
{user, machine} = AuthUtils.get_api_details(conn)
{:error, :generic, reason} ->
conn |> put_status(400) |> json(%{"error" => reason})
with {:ok, %DateTime{} = datetime} <- parse_timestamp(timestamp),
{:ok, datetime} <- check_datetime_diff(datetime),
{:ok, %Pulse{} = pulse} <- create_pulse(user, machine, datetime),
:ok <- create_xps(pulse, xps) do
:ok
{:error, :internal, reason} ->
conn |> put_status(500) |> json(%{"error" => reason})
end
end
end
defp parse_timestamp(timestamp) do
......@@ -79,20 +75,21 @@ defmodule CodeStats.PulseController do
|> Repo.insert()
|> case do
{:ok, %Pulse{} = pulse} -> {:ok, pulse}
{:error, changeset} -> {:error, :generic, "Could not create pulse: #{inspect changeset.errors}"}
{:error, _} -> {:error, :generic, "Could not create pulse because of an unknown issue."}
end
end
defp create_xps(pulse, xps) do
try do
Enum.each(xps, fn
inserted_xps = Enum.map(xps, fn
%{"language" => language, "xp" => xp} when is_integer(xp) ->
case create_xp(pulse, language, xp) do
:ok -> :ok
{:ok, inserted_xp} -> inserted_xp
{:error, _, reason} -> raise reason
end
_ -> raise "Invalid XP format."
end)
{:ok, inserted_xps}
rescue
e in RuntimeError -> {:error, :generic, e.message}
end
......@@ -107,8 +104,12 @@ defmodule CodeStats.PulseController do
|> Changeset.put_change(:language_id, language.id)
|> Repo.insert()
|> case do
{:ok, _} -> :ok
{:error, changeset} -> {:error, :generic, "Could not create XP: #{inspect changeset.errors}"}
{:ok, inserted_xp} ->
# Set the language so that it can be used in update_caches
inserted_xp = %{inserted_xp | language: language}
{:ok, inserted_xp}
{:error, _} -> {:error, :generic, "Could not create XP because of an unknown issue."}
end
end
end
......@@ -130,9 +131,19 @@ defmodule CodeStats.PulseController do
{:error, _} ->
case Repo.one(get_query) do
%Language{} = language -> {:ok, language}
nil -> {:error, :internal, "Could not get-create-get language: #{language_name}"}
nil -> {:error, :internal, "Could not get-create-get language because of an unknown issue."}
end
end
end
end
defp update_caches(xps) do
try do
Enum.each(xps, fn xp ->
CacheService.add_total_language_xp(xp.language, xp.amount)
end)
rescue
e in RuntimeError -> {:error, :generic, e.message}
end
end
end
......@@ -29,91 +29,85 @@
<h2>Changelog</h2>
<h3>1.4.0 – 2016-07-16 – First performance update</h3>
<ul>
<li>Total XP and most popular languages counters on the front page are now in-memory, stored inside ETS. They are updated on every incoming pulse (for now) and totally refreshed every 15 minutes. This should make the front page much faster.</li>
<li>Added an index to make "XP in last 12 hours" queries faster both on front page and profile pages.</li>
<li>Accessibility was improved with regards to dates and progress bars.</li>
<li>Fixed average XP per day display, which showed XP in scientific format in some cases.</li>
<li>Added real page titles for better SEO and bookmarking.</li>
<li>Updated the codebase for Elixir 1.3, Phoenix 1.2 and Ecto 2.0.</li>
<li>Fixed some JSON errors not being sent correctly in the API.</li>
</ul>
<h3>1.3.7 – 2016-06-08 – Fix username uniqueness</h3>
<p>
<ul>
<li>Fix username uniqueness, due to a problem with SQL migrations a unique index was not created and multiple users could be registered with the same username. 😱</li>
<li>Change some IDs to bigint because if it's worth doing, it's worth overdoing.</li>
<li>Remove old unused total XP property from users.</li>
<li>Clarified git repo licences a bit.</li>
</ul>
</p>
<ul>
<li>Fix username uniqueness, due to a problem with SQL migrations a unique index was not created and multiple users could be registered with the same username. 😱</li>
<li>Change some IDs to bigint because if it's worth doing, it's worth overdoing.</li>
<li>Remove old unused total XP property from users.</li>
<li>Clarified git repo licences a bit.</li>
</ul>
<h3>1.3.3–1.3.6 – 2016-06-07 – JetBrains and username path fixes</h3>
<p>
<ul>
<li>Added details of the published IntelliJ/JetBrains plugin.</li>
<li>Fixed spaces in usernames resulting in unreachable profile pages.</li>
<li>Disallowed the plus sign in usernames to avoid conflicts with spaces.</li>
</ul>
</p>
<ul>
<li>Added details of the published IntelliJ/JetBrains plugin.</li>
<li>Fixed spaces in usernames resulting in unreachable profile pages.</li>
<li>Disallowed the plus sign in usernames to avoid conflicts with spaces.</li>
</ul>
<h3>1.3.2 – 2016-06-06 – JSON errors</h3>
<p>
<ul>
<li>Enabled JSON errors for easier debugging of 500 Internal server errors.</li>
</ul>
</p>
<ul>
<li>Enabled JSON errors for easier debugging of 500 Internal server errors.</li>
</ul>
<h3>1.3.1 – 2016-06-05 – Language name fix</h3>
<p>
<ul>
<li>Fixed languages being created with different capitalisations. Now language names are case insensitive.</li>
</ul>
</p>
<ul>
<li>Fixed languages being created with different capitalisations. Now language names are case insensitive.</li>
</ul>
<h3>1.3.0 – 2016-06-04 – Machine stats</h3>
<p>
<ul>
<li>Added machine statistics to profile page.</li>
<li>Added recalculation of XP once a day to fix XP errors in profile view.</li>
<li>Added favicons.</li>
<li>Added more user info to profile view.</li>
</ul>
</p>
<ul>
<li>Added machine statistics to profile page.</li>
<li>Added recalculation of XP once a day to fix XP errors in profile view.</li>
<li>Added favicons.</li>
<li>Added more user info to profile view.</li>
</ul>
<h3>1.2.2 – 2016-06-02 – Vincit Oy</h3>
<p>
<ul>
<li>Added note about my company who have sponsored the development of the service.</li>
<li>Added more information to the plugins page.</li>
</ul>
</p>
<ul>
<li>Added note about my company who have sponsored the development of the service.</li>
<li>Added more information to the plugins page.</li>
</ul>
<h3>1.2.1 – 2016-05-31 – Small fixes</h3>
<p>
<ul>
<li>Fix <a href="https://github.com/Nicd/code-stats/issues/1">#1</a>: coded_at time range not documented in API docs</li>
<li>Fix <a href="https://github.com/Nicd/code-stats/issues/2">#2</a>: Endpoint path typo in API docs</li>
<li>Fix <a href="https://github.com/Nicd/code-stats/issues/4">#4</a>: Refactor README</li>
<li>Fix <a href="https://github.com/Nicd/code-stats/issues/5">#5</a>: Add link to example stats to front page</li>
<li>Fix <a href="https://github.com/Nicd/code-stats/issues/6">#6</a>: Add clarification about recent XP</li>
</ul>
</p>
<ul>
<li>Fix <a href="https://github.com/Nicd/code-stats/issues/1">#1</a>: coded_at time range not documented in API docs</li>
<li>Fix <a href="https://github.com/Nicd/code-stats/issues/2">#2</a>: Endpoint path typo in API docs</li>
<li>Fix <a href="https://github.com/Nicd/code-stats/issues/4">#4</a>: Refactor README</li>
<li>Fix <a href="https://github.com/Nicd/code-stats/issues/5">#5</a>: Add link to example stats to front page</li>
<li>Fix <a href="https://github.com/Nicd/code-stats/issues/6">#6</a>: Add clarification about recent XP</li>
</ul>
<h3>1.2.0 – 2016-05-31 – Scaling back levels</h3>
<p>
<ul>
<li>Scaled back the level algorithm to grant less levels. XP is not affected. The aim is to prevent an inflation of levels. The level algorithm may be modified further later on.</li>
<li>Combined the source and changes page.</li>
</ul>
</p>
<ul>
<li>Scaled back the level algorithm to grant less levels. XP is not affected. The aim is to prevent an inflation of levels. The level algorithm may be modified further later on.</li>
<li>Combined the source and changes page.</li>
</ul>
<h3>1.1.1 – 2016-05-30 – Clock drift</h3>
<p>
<ul>
<li>Fixed a case where clock drift either on the client or the server caused pulses to not be accepted.</li>
</ul>
</p>
<ul>
<li>Fixed a case where clock drift either on the client or the server caused pulses to not be accepted.</li>
</ul>
<h3>1.1.0 – 2016-05-30 – First public release</h3>
......
......@@ -30,17 +30,15 @@
<li>
Programming since
<time datetime="<%= Calendar.Strftime.strftime!(@user.inserted_at, "%F") %>">
<%= Calendar.Strftime.strftime!(@user.inserted_at, "%b %e %Y") %>
</time>.
<%= Calendar.Strftime.strftime!(@user.inserted_at, "%b %e, %Y") %></time>.
</li>
<li>Average <%= format_xp(@xp_per_day) %>&nbsp;XP per day.</li>
<li>
Last coded
<%= if @last_day_coded != nil do %>
at
on
<time datetime="<%= Calendar.Strftime.strftime!(@last_day_coded, "%F") %>">
<%= Calendar.Strftime.strftime!(@last_day_coded, "") %>
</time>.
<%= Calendar.Strftime.strftime!(@last_day_coded, "%b %e, %Y") %></time>.
<% else %>
<em>never</em>.
<% end %>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment