Commit ebf658d3 authored by JoeBew42's avatar JoeBew42

fixes #32: expose an API for pulses data export

this commit will add a new API /my/pulses that is available for all authenticated users to export all data related to its pulses in a CSV format. example of the request in order to get the exported data: curl -H "accept: text/csv" http://<code-stats-url>/my/pulses
parent a34735d1
......@@ -20,7 +20,12 @@ These are current targets, older versions _might_ work:
### First time install
```
# Database setup for development environment
Set up PostgreSQL matching the settings in `config/dev.exs`.
# First create empty versions of required config files for it to compile
echo use Mix.Config > config/{appsignal,dev.secret}.exs
# Then:
......@@ -37,6 +42,7 @@ nano config/dev.secret.exs # Set up dev config with at least the line "use Mix.
### Commands
* `mix test`: Run all tests suite
* `mix phx.server`: Run development server on port 5000 (default, you can configure this in
`dev.secret.exs`)
* `mix frontend.build`: Build the JS/CSS frontend
......
......@@ -3702,6 +3702,15 @@
"integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=",
"dev": true
},
"string_decoder": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.1.tgz",
"integrity": "sha1-YuIA8DmVWmgQ2N8KM//A8BNmLZg=",
"dev": true,
"requires": {
"safe-buffer": "5.1.0"
}
},
"string-width": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
......@@ -3713,15 +3722,6 @@
"strip-ansi": "3.0.1"
}
},
"string_decoder": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.1.tgz",
"integrity": "sha1-YuIA8DmVWmgQ2N8KM//A8BNmLZg=",
"dev": true,
"requires": {
"safe-buffer": "5.1.0"
}
},
"stringstream": {
"version": "0.0.5",
"resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz",
......
......@@ -2,6 +2,7 @@ defmodule CodeStats.User.Pulse do
use Ecto.Schema
import Ecto.Changeset
import Ecto.Query
schema "pulses" do
# When the Pulse was generated on the client. This is somewhat confusingly named
......@@ -33,4 +34,10 @@ defmodule CodeStats.User.Pulse do
|> cast(params, [:sent_at, :sent_at_local, :tz_offset])
|> validate_required([:sent_at, :sent_at_local, :tz_offset])
end
def xps_by_user_id(id) do
__MODULE__
|> where([p], p.user_id == ^id)
|> preload([:machine, {:xps, [:language]}])
end
end
......@@ -12,6 +12,8 @@ defmodule CodeStatsWeb.PulseController do
alias CodeStatsWeb.ProfileChannel
alias CodeStatsWeb.FrontpageChannel
alias CodeStatsWeb.GeoIPPlug
alias CodeStatsWeb.Utils.CsvFormatter
alias CodeStats.Repo
alias CodeStats.Language
alias CodeStats.Language.CacheService
......@@ -20,6 +22,18 @@ defmodule CodeStatsWeb.PulseController do
plug GeoIPPlug
def list(conn, _params) do
csv = AuthUtils.get_current_user_id(conn)
|> Pulse.xps_by_user_id
|> Repo.all
|> CsvFormatter.format(["sent_at", "sent_at_local", "tz_offset", "languages", "machine", "xps"])
conn
|> put_resp_content_type("text/csv")
|> put_resp_header("content-disposition", "attachment; filename=\"pulses.csv\"")
|> send_resp(200, csv)
end
def add(conn, %{"coded_at" => timestamp, "xps" => xps}) when is_list(xps) do
{user, machine} = AuthUtils.get_api_details(conn)
......
defmodule CodeStatsWeb.Router do
use CodeStatsWeb, :router
pipeline :browser do
plug :accepts, ["html"]
pipeline :browser_common do
plug :fetch_session
plug :fetch_flash
plug :protect_from_forgery
......@@ -11,6 +10,14 @@ defmodule CodeStatsWeb.Router do
plug CodeStatsWeb.SetSessionUserPlug
end
pipeline :browser_html do
plug :accepts, ["html"]
end
pipeline :browser_csv do
plug :accepts, ["csv"]
end
pipeline :browser_auth do
plug CodeStatsWeb.AuthRequiredPlug
end
......@@ -28,7 +35,8 @@ defmodule CodeStatsWeb.Router do
end
scope "/", CodeStatsWeb do
pipe_through :browser # Use the default browser stack
pipe_through :browser_html
pipe_through :browser_common
get "/", PageController, :index
......@@ -80,6 +88,14 @@ defmodule CodeStatsWeb.Router do
end
end
scope "/", CodeStatsWeb do
pipe_through :browser_csv
pipe_through :browser_common
pipe_through :browser_auth
get "/my/pulses", PulseController, :list
end
scope "/api", CodeStatsWeb do
pipe_through :api
......@@ -87,7 +103,6 @@ defmodule CodeStatsWeb.Router do
scope "/my" do
pipe_through :api_auth
post "/pulses", PulseController, :add
end
end
......
defmodule CodeStatsWeb.Utils.CsvFormatter do
alias CodeStats.User.Pulse
@doc """
Returns a semi-colon separated values
pulses = [%Pulse{sent_at: sent_at, sent_at_local: sent_at_local, tz_offset: tz_offset, machine: machine, xps: xps}]
"""
def format([], headers), do: Enum.join(headers, ";") <> "\n"
def format(pulses, headers) do
pulses
|> Enum.map(&to_list(&1))
|> prepend(headers)
|> CSV.encode(separator: ?;, delimiter: "\n")
|> Enum.to_list()
|> to_string()
end
defp to_list(%Pulse{sent_at: sent_at, sent_at_local: sent_at_local, tz_offset: tz_offset, machine: machine, xps: xps}) do
[
"#{sent_at}",
"#{sent_at_local}",
"#{tz_offset}",
xps |> Enum.map(fn xp -> xp.language.name end) |> Enum.uniq() |> Enum.join(","),
machine.name,
Enum.reduce(xps, 0, fn xp, acc -> acc + xp.amount end)
]
end
defp prepend(list, headers), do: [headers | list]
end
......@@ -66,7 +66,8 @@ defmodule CodeStats.Mixfile do
{:geolix, "~> 0.14.0"},
{:geolite2data, "~> 0.0.3"},
{:remote_ip, "~> 0.1.3"},
{:distillery, git: "https://github.com/bitwalker/distillery.git", ref: "67905e230ce0e861a739756c1f79ba9124c5fd3e", runtime: false}
{:distillery, git: "https://github.com/bitwalker/distillery.git", ref: "67905e230ce0e861a739756c1f79ba9124c5fd3e", runtime: false},
{:csv, "~> 2.0.0"}
]
end
......
......@@ -9,6 +9,7 @@
"corsica": {:hex, :corsica, "1.0.0", "e11d39e72c9907c96650d1a5b5586e9f333643a17d0120380ad1d7dacdc9eb43", [:mix], [{:cowboy, "~> 1.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
"cowboy": {:hex, :cowboy, "1.1.2", "61ac29ea970389a88eca5a65601460162d370a70018afe6f949a29dca91f3bb0", [:rebar3], [{:cowlib, "~> 1.0.2", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3.2", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"},
"cowlib": {:hex, :cowlib, "1.0.2", "9d769a1d062c9c3ac753096f868ca121e2730b9a377de23dec0f7e08b1df84ee", [:make], [], "hexpm"},
"csv": {:hex, :csv, "2.0.0", "c66fea89ba7862b94901baf0871285e9b73cad89c5fdb57a6386d2adcf29593e", [:mix], [{:parallel_stream, "~> 1.0.4", [hex: :parallel_stream, repo: "hexpm", optional: false]}], "hexpm"},
"db_connection": {:hex, :db_connection, "1.1.2", "2865c2a4bae0714e2213a0ce60a1b12d76a6efba0c51fbda59c9ab8d1accc7a8", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, repo: "hexpm", optional: true]}], "hexpm"},
"decimal": {:hex, :decimal, "1.4.0", "fac965ce71a46aab53d3a6ce45662806bdd708a4a95a65cde8a12eb0124a1333", [:mix], [], "hexpm"},
"decimal_arithmetic": {:hex, :decimal_arithmetic, "0.1.1", "15f651ace567474d1d332f3cbf55e9203acb296a26591aa0f3fee346de4ff775", [:mix], [{:decimal, "~> 1.1", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm"},
......@@ -31,6 +32,7 @@
"mime": {:hex, :mime, "1.1.0", "01c1d6f4083d8aa5c7b8c246ade95139620ef8effb009edde934e0ec3b28090a", [:mix], [], "hexpm"},
"mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], [], "hexpm"},
"number": {:hex, :number, "0.5.4", "221ee2988dc3abaa3bd4dce43cbe603f20c8ca7343fc160769434fddd19c6fc0", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:decimal_arithmetic, "~> 0.1", [hex: :decimal_arithmetic, repo: "hexpm", optional: false]}], "hexpm"},
"parallel_stream": {:hex, :parallel_stream, "1.0.6", "b967be2b23f0f6787fab7ed681b4c45a215a81481fb62b01a5b750fa8f30f76c", [:mix], [], "hexpm"},
"phoenix": {:hex, :phoenix, "1.3.0", "1c01124caa1b4a7af46f2050ff11b267baa3edb441b45dbf243e979cd4c5891b", [:mix], [{:cowboy, "~> 1.0", [hex: :cowboy, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.3.3 or ~> 1.4", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"},
"phoenix_ecto": {:hex, :phoenix_ecto, "3.3.0", "702f6e164512853d29f9d20763493f2b3bcfcb44f118af2bc37bb95d0801b480", [:mix], [{:ecto, "~> 2.1", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
"phoenix_html": {:hex, :phoenix_html, "2.10.4", "d4f99c32d5dc4918b531fdf163e1fd7cf20acdd7703f16f5d02d4db36de803b7", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
......
defmodule CodeStatsWeb.PulseControllerTest do
use CodeStatsWeb.ConnCase
alias CodeStats.User
alias CodeStats.User.Machine
alias CodeStats.User.Pulse
alias CodeStats.XP
alias CodeStats.Language
alias CodeStatsWeb.AuthUtils
describe "as a not authenticated user" do
test "GET /my/pulses should return 403 forbidden", %{conn: conn} do
conn = conn
|> put_req_header("accept", "text/csv")
|> get("/my/pulses")
assert conn.status == 403
end
end
describe "as an authenticated user with some pulses" do
setup do
{:ok, language} = Language.get_or_create("elixir")
{:ok, user} = create_user("user@somewhere", "test_user")
{:ok, another_user} = create_user("another_user@somewhere", "another_test_user")
create_data_for(user, language)
create_data_for(another_user, language)
%{user: user}
end
test "GET /my/pulses should export data in CSV", %{conn: conn, user: user} do
conn = conn
|> authenticated_as(user)
|> put_req_header("accept", "text/csv")
|> get("/my/pulses")
assert conn.status == 200
assert conn.resp_headers |> contains?("content-type", "text/csv; charset=utf-8")
assert conn.resp_headers |> contains?("content-disposition", "attachment; filename=\"pulses.csv\"")
assert conn.resp_body =~ "sent_at;sent_at_local;tz_offset;languages;machine;xps\n"
assert conn.resp_body =~ "2017-11-27 23:00:00.000000Z;2017-11-28 00:00:00.000000;60;elixir;test_machine;1\n"
end
end
describe "as an authenticated user with no pulses" do
setup do
{:ok, user} = create_user("user@somewhere", "test_user")
%{user: user}
end
test "GET /my/pulses should return an empty CSV", %{conn: conn, user: user} do
conn = conn
|> authenticated_as(user)
|> put_req_header("accept", "text/csv")
|> get("/my/pulses")
assert conn.status == 200
assert conn.resp_headers |> contains?("content-type", "text/csv; charset=utf-8")
assert conn.resp_headers |> contains?("content-disposition", "attachment; filename=\"pulses.csv\"")
assert conn.resp_body == "sent_at;sent_at_local;tz_offset;languages;machine;xps\n"
end
end
defp authenticated_as(conn, user) do
conn
|> init_test_session
|> AuthUtils.force_auth_user_id(user.id)
end
# this should be replaced with the init_test_session
# of the new Plug. Maybe it is useful to bump the
# phoenix version
# See: https://hexdocs.pm/plug/Plug.Test.html#init_test_session/2
defp init_test_session(conn, _session \\ %{}) do
conn
|> put_private(:plug_session, %{})
|> put_private(:plug_session_fetch, :done)
end
defp contains?(headers, key, value), do: Enum.member?(headers, {key, value})
defp create_user(email, username) do
%User{email: email, username: username, password: "test_password"}
|> User.changeset(%{})
|> Repo.insert
end
defp create_data_for(user, language) do
{:ok, machine} = %Machine{name: "test_machine"}
|> Machine.changeset(%{})
|> Ecto.Changeset.put_change(:user_id, user.id)
|> Repo.insert
{:ok, sent_at} = Calendar.DateTime.from_erl({{2017,11,27},{23,00,00}}, "Etc/UTC")
local_datetime = Calendar.DateTime.add!(sent_at, 3600) |> Calendar.DateTime.to_naive()
{:ok, pulse} = Pulse.changeset(%Pulse{sent_at: sent_at, tz_offset: 60, sent_at_local: local_datetime}, %{})
|> Ecto.Changeset.put_change(:user_id, user.id)
|> Ecto.Changeset.put_change(:machine_id, machine.id)
|> Repo.insert
{:ok, _} = XP.changeset(%XP{amount: 1})
|> Ecto.Changeset.put_change(:pulse_id, pulse.id)
|> Ecto.Changeset.put_change(:language_id, language.id)
|> Ecto.Changeset.put_change(:original_language_id, language.id)
|> Repo.insert
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment