Commit 68e705ca authored by Mikko Ahlroth's avatar Mikko Ahlroth
Browse files

Initial commit

parents
Pipeline #33043322 failed with stages
in 1 minute and 14 seconds
# Used by "mix format"
[
inputs: ["mix.exs", "{config,lib,test}/**/*.{ex,exs}"]
]
# The directory Mix will write compiled artifacts to.
/_build/
# If you run "mix test --cover", coverage assets end up here.
/cover/
# The directory Mix downloads your dependencies sources to.
/deps/
# Where 3rd-party dependencies like ExDoc output generated docs.
/doc/
# Ignore .fetch files in case you like to edit your project deps locally.
/.fetch
# If the VM crashes, it generates a dump, let's ignore it too.
erl_crash.dump
# Also ignore archive artifacts (built via "mix archive.build").
*.ez
# Ignore package tarball (built via "mix hex.build").
mebe_2-*.tar
# Ignore secret configz
/config/*.secret.exs
/data
/.elixir_ls
# Built frontend assets
/priv/static
[
{
"files": "**/*.{ex,exs}",
"command": "mix format ${srcFile}"
}
]
# Mebe2
**TODO: Add description**
## Installation
If [available in Hex](https://hex.pm/docs/publish), the package can be installed
by adding `mebe_2` to your list of dependencies in `mix.exs`:
```elixir
def deps do
[
{:mebe_2, "~> 0.1.0"}
]
end
```
Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc)
and published on [HexDocs](https://hexdocs.pm). Once published, the docs can
be found at [https://hexdocs.pm/mebe_2](https://hexdocs.pm/mebe_2).
# This file is responsible for configuring your application
# and its dependencies with the aid of the Mix.Config module.
use Mix.Config
# This configuration is loaded before any dependency and is restricted
# to this project. If another project depends on this project, this
# file won't be loaded nor affect the parent project. For this reason,
# if you want to provide default values for your application for
# 3rd-party users, it should be done in your "mix.exs" file.
# You can configure your application as:
#
# config :mebe_2, key: :value
#
# and access this configuration in your application as:
#
# Application.get_env(:mebe_2, :key)
#
# You can also configure a 3rd-party app:
#
# config :logger, level: :info
#
config :mebe_2,
# The path to crawl post and page data from. No trailing slash, use an absolute path.
data_path: Path.expand("data"),
# Port to listen on
port: 2124,
# Basic blog information
blog_name: "My awesome blog",
blog_author: "Author McAuthor",
# Absolute URL to the site, including protocol, no trailing slash
absolute_url: "http://localhost:2124",
# Set to true to show author header from posts, if available (blog_author will be used as default)
multi_author_mode: false,
# If multi author mode is on, use blog_author as default author (if this is false, no author will be set if post has no author header)
use_default_author: true,
# Default timezone to use for posts with time data
time_default_tz: "Europe/Helsinki",
# Force "Read more…" text to display even if there is no more content
force_read_more: false,
# Set to true to enable RSS feeds
enable_feeds: false,
# Show full content in feeds instead of short content
feeds_full_content: false,
posts_per_page: 10,
posts_in_feed: 20,
# Disqus comments
# Use Disqus comments
disqus_comments: false,
# Show comments for pages too
page_commenting: false,
disqus_shortname: "my-awesome-blog",
# Extra HTML that is injected to every page, right before </body>. Useful for analytics scripts.
extra_html: """
<script>
window.tilastokeskus_url = '/tilastokeskus/track';
</script>
<script src="/tilastokeskus/track.js" async></script>
"""
if Mix.env() == :dev do
config :exsync, :extensions, [".ex", ".eex"]
end
# If you wish to compile in secret settings, use the following file. Note that the settings in
# the file will be set at release generation time and cannot be changed later.
if File.exists?("config/config.secret.exs") do
import_config("config.secret.exs")
end
defmodule Mebe2.Engine.Crawler do
@moduledoc """
The crawler goes through the specified directory, opening and parsing all the matching files
inside concurrently.
"""
require Logger
alias Mebe2.Engine.{Parser, Utils, SlugUtils}
alias Mebe2.Engine.Models.{Page, Post}
def crawl(path) do
get_files(path)
|> Enum.map(fn file -> Task.async(__MODULE__, :parse, [file]) end)
|> handle_responses()
|> construct_archives()
end
def get_files(path) do
path = path <> "/**/*.md"
Logger.info("Searching files using '#{path}' with cwd '#{System.cwd()}'")
files = Path.wildcard(path)
Logger.info("Found files:")
for file <- files do
Logger.info(file)
end
files
end
def parse(file) do
try do
File.read!(file)
|> Parser.parse(Path.basename(file))
rescue
_ ->
Logger.error("Could not parse file #{file}. Exception: #{inspect(__STACKTRACE__)}")
:error
end
end
def handle_responses(tasklist) do
Enum.map(tasklist, fn task -> Task.await(task) end)
end
def construct_archives(datalist) do
Enum.reduce(
datalist,
%{
pages: %{},
posts: [],
years: %{},
months: %{},
tags: %{},
authors: %{},
author_names: %{}
},
fn pagedata, acc ->
case pagedata do
# Ignore pages/posts that could not be parsed
:error ->
acc
%Page{} ->
put_in(acc, [:pages, pagedata.slug], pagedata)
%Post{} ->
{{year, month, _}, _} = Calendar.DateTime.to_erl(pagedata.datetime)
tags =
Enum.reduce(pagedata.tags, acc.tags, fn tag, tagmap ->
posts = Map.get(tagmap, tag, [])
Map.put(tagmap, tag, [pagedata | posts])
end)
{authors, author_names} = form_authors(acc, pagedata)
year_posts = [pagedata | Map.get(acc.years, year, [])]
month_posts = [pagedata | Map.get(acc.months, {year, month}, [])]
%{
acc
| posts: [pagedata | acc.posts],
years: Map.put(acc.years, year, year_posts),
months: Map.put(acc.months, {year, month}, month_posts),
tags: tags,
authors: authors,
author_names: author_names
}
end
end
)
end
defp form_authors(datalist, pagedata) do
multi_author_mode = Mebe2.get_conf(:multi_author_mode)
do_form_authors(multi_author_mode, datalist, pagedata)
end
defp do_form_authors(false, _, _), do: {%{}, %{}}
defp do_form_authors(true, %{authors: authors, author_names: author_names}, pagedata) do
author_name = Utils.get_author(pagedata)
author_slug = SlugUtils.slugify(author_name)
author_posts = [pagedata | Map.get(authors, author_slug, [])]
authors = Map.put(authors, author_slug, author_posts)
# Authors end up with the name that was in the post with the first matching slug
author_names = Map.put_new(author_names, author_slug, author_name)
{authors, author_names}
end
end
defmodule Mebe2.Engine.DB do
require Logger
alias Mebe2.Engine.{Utils, SlugUtils, Models}
alias Calendar.DateTime
@moduledoc """
Stuff related to storing the blog data to memory (ETS).
"""
# Table for meta information, like the counts of posts and names
# of authors
@meta_table :mebe2_meta
# Table for storing pages by slug
@page_table :mebe2_pages
# Table for sequential retrieval of posts (for list pages)
@post_table :mebe2_posts
# Table for quick retrieval of single post (with key)
@single_post_table :mebe2_single_posts
# Table for storing posts with tag as first element of key
@tag_table :mebe2_tags
# Table for storing posts by specific authors
@author_table :mebe2_authors
# Table for storing menu data
@menu_table :mebe2_menu
@spec init() :: :ok
def init() do
# Only create tables if they don't exist already
if :ets.info(@meta_table) == :undefined do
:ets.new(@meta_table, [:named_table, :set, :protected, read_concurrency: true])
:ets.new(@page_table, [:named_table, :set, :protected, read_concurrency: true])
:ets.new(@post_table, [:named_table, :ordered_set, :protected, read_concurrency: true])
:ets.new(@single_post_table, [:named_table, :set, :protected, read_concurrency: true])
:ets.new(@tag_table, [:named_table, :ordered_set, :protected, read_concurrency: true])
:ets.new(@menu_table, [:named_table, :ordered_set, :protected, read_concurrency: true])
if Mebe2.get_conf(:multi_author_mode) do
:ets.new(@author_table, [:named_table, :ordered_set, :protected, read_concurrency: true])
end
end
:ok
end
@spec destroy() :: :ok
def destroy() do
:ets.delete_all_objects(@meta_table)
:ets.delete_all_objects(@page_table)
:ets.delete_all_objects(@post_table)
:ets.delete_all_objects(@single_post_table)
:ets.delete_all_objects(@tag_table)
:ok
end
@spec insert_count(:all, integer) :: true
def insert_count(:all, count) do
insert_meta(:all, :all, count)
end
@spec insert_count(atom, String.t() | integer, integer) :: true
def insert_count(type, key, count) do
insert_meta(type, key, count)
end
@spec insert_menu([{String.t(), String.t()}]) :: true
def insert_menu(menu) do
# Format for ETS because it needs a tuple
menu = Enum.map(menu, fn menuitem -> {menuitem.slug, menuitem} end)
:ets.insert(@menu_table, menu)
end
@spec insert_posts([Models.Post.t()]) :: :ok
def insert_posts(posts) do
ordered_posts =
Enum.map(posts, fn post ->
{{year, month, day}, _} = DateTime.to_erl(post.datetime)
{{year, month, day, post.order}, post}
end)
single_posts =
Enum.map(posts, fn post ->
{{year, month, day}, _} = DateTime.to_erl(post.datetime)
{{year, month, day, post.slug}, post}
end)
:ets.insert(@post_table, ordered_posts)
:ets.insert(@single_post_table, single_posts)
if Mebe2.get_conf(:multi_author_mode) do
author_posts =
Enum.filter(posts, fn post -> Map.has_key?(post.extra_headers, "author") end)
|> Enum.map(fn post ->
{{year, month, day}, _} = DateTime.to_erl(post.datetime)
author_slug = Utils.get_author(post) |> SlugUtils.slugify()
{{author_slug, year, month, day, post.order}, post}
end)
:ets.insert(@author_table, author_posts)
end
:ok
end
@spec insert_page(Models.Page.t()) :: true
def insert_page(page) do
:ets.insert(@page_table, {page.slug, page})
end
@spec insert_tag_posts(%{optional(String.t()) => Models.Post.t()}) :: true
def insert_tag_posts(tags) do
tag_posts =
Enum.reduce(Map.keys(tags), [], fn tag, acc ->
Enum.reduce(tags[tag], acc, fn post, inner_acc ->
{{year, month, day}, _} = DateTime.to_erl(post.datetime)
[{{tag, year, month, day, post.order}, post} | inner_acc]
end)
end)
:ets.insert(@tag_table, tag_posts)
end
@spec insert_author_posts(%{optional(String.t()) => Models.Post.t()}) :: true
def insert_author_posts(authors) do
author_posts =
Enum.reduce(Map.keys(authors), [], fn author_slug, acc ->
Enum.reduce(authors[author_slug], acc, fn post, inner_acc ->
{{year, month, day}, _} = DateTime.to_erl(post.datetime)
[{{author_slug, year, month, day, post.order}, post} | inner_acc]
end)
end)
:ets.insert(@author_table, author_posts)
end
@spec insert_author_names(%{optional(String.t()) => String.t()}) :: true
def insert_author_names(author_names_map) do
author_names =
Enum.reduce(Map.keys(author_names_map), [], fn author_slug, acc ->
[{{:author_name, author_slug}, author_names_map[author_slug]} | acc]
end)
:ets.insert(@meta_table, author_names)
end
@spec get_menu() :: [Models.MenuItem.t()]
def get_menu() do
case :ets.match(@menu_table, :"$1") do
[] -> []
results -> format_menu(results)
end
end
@spec get_reg_posts(integer(), integer()) :: [Models.Post.t()]
def get_reg_posts(first, last) do
get_post_list(@post_table, [{:"$1", [], [:"$_"]}], first, last)
end
@spec get_tag_posts(String.t(), integer(), integer()) :: [Models.Post.t()]
def get_tag_posts(tag, first, last) do
get_post_list(@tag_table, [{{{tag, :_, :_, :_, :_}, :"$1"}, [], [:"$_"]}], first, last)
end
@spec get_author_posts(String.t(), integer(), integer()) :: [Models.Post.t()]
def get_author_posts(author_slug, first, last) do
get_post_list(
@author_table,
[{{{author_slug, :_, :_, :_, :_}, :"$1"}, [], [:"$_"]}],
first,
last
)
end
@spec get_year_posts(integer(), integer(), integer()) :: [Models.Post.t()]
def get_year_posts(year, first, last) do
get_post_list(@post_table, [{{{year, :_, :_, :_}, :"$1"}, [], [:"$_"]}], first, last)
end
@spec get_month_posts(integer(), integer(), integer(), integer()) :: [Models.Post.t()]
def get_month_posts(year, month, first, last) do
get_post_list(@post_table, [{{{year, month, :_, :_}, :"$1"}, [], [:"$_"]}], first, last)
end
@spec get_page(String.t()) :: Models.Page.t() | nil
def get_page(slug) do
case :ets.match_object(@page_table, {slug, :"$1"}) do
[{_, page}] -> page
_ -> nil
end
end
@spec get_post(integer(), integer(), integer(), String.t()) :: Models.Post.t() | nil
def get_post(year, month, day, slug) do
case :ets.match_object(@single_post_table, {{year, month, day, slug}, :"$1"}) do
[{_, post}] -> post
_ -> nil
end
end
@spec get_count(:all) :: integer()
def get_count(:all) do
get_count(:all, :all)
end
@spec get_count(atom, :all | integer | String.t()) :: integer()
def get_count(type, key) do
get_meta(type, key, 0)
end
@spec get_author_name(String.t()) :: String.t()
def get_author_name(author_slug) do
get_meta(:author_name, author_slug, author_slug)
end
@spec insert_meta(atom, :all | integer | String.t(), integer | String.t()) :: true
defp insert_meta(type, key, value) do
:ets.insert(@meta_table, {{type, key}, value})
end
@spec get_meta(atom, :all | integer | String.t(), integer | String.t()) :: integer | String.t()
defp get_meta(type, key, default) do
case :ets.match_object(@meta_table, {{type, key}, :"$1"}) do
[{{_, _}, value}] -> value
[] -> default
end
end
# Combine error handling of different post listing functions
@spec get_post_list(atom, [tuple], integer, integer) :: [Models.Post.t()]
defp get_post_list(table, matchspec, first, last) do
case :ets.select_reverse(table, matchspec, first + last) do
:"$end_of_table" ->
[]
{result, _} ->
Enum.split(result, first) |> elem(1) |> ets_to_data()
end
end
# Remove key from data returned from ETS
@spec ets_to_data([{any, any}]) :: any
defp ets_to_data(data) do
for {_, actual} <- data, do: actual
end
# Format menu results (convert [{slug, %MenuItem{}}] to %MenuItem{})
@spec format_menu([[{String.t(), Models.MenuItem.t()}]]) :: [Models.MenuItem.t()]
defp format_menu(results) do
for [{_, result}] <- results, do: result
end
end
defmodule Mebe2.Engine.MenuParser do
@moduledoc """
This module handles the parsing of the menu file, which lists the links in the menu bar.
"""
alias Mebe2.Engine.Models.MenuItem
def parse(data_path) do
(data_path <> "/menu")
|> File.read!()
|> split_lines
|> parse_lines
|> Enum.filter(fn item -> item != nil end)
end
defp split_lines(menudata) do
String.split(menudata, ~R/\r?\n/)
end
defp parse_lines(menulines) do
for line <- menulines do
case String.split(line, " ") do
[_] -> nil
[link | rest] -> %MenuItem{slug: link, title: Enum.join(rest, " ")}
end
end
end
end
defmodule Mebe2.Engine.Models do
@moduledoc """
This module contains the data models of the blog engine.
"""
defmodule PageData do
defstruct filename: nil,
title: nil,
headers: [],
content: nil
@type t :: %__MODULE__{
filename: String.t(),
title: String.t(),
headers: [{String.t(), String.t()}],
content: String.t()
}
end
defmodule Post do
defstruct slug: nil,
title: nil,
datetime: nil,
time_given: false,
tags: [],
content: nil,
short_content: nil,
order: 0,
has_more: false,
extra_headers: %{}
@type t :: %__MODULE__{
slug: String.t(),
title: String.t(),
datetime: DateTime.t(),
time_given: boolean,
tags: [String.t()],
content: String.t(),
short_content: String.t(),
order: integer,
has_more: boolean,
extra_headers: %{optional(String.t()) => String.t()}
}
end
defmodule Page do
defstruct slug: nil,
title: nil,
content: nil,
extra_headers: %{}
@type t :: %__MODULE__{
slug: String.t(),
title: String.t(),
content: String.t(),
extra_headers: %{optional(String.t()) => String.t()}
}
end
defmodule MenuItem do
defstruct slug: nil,
title: nil
@type t :: %__MODULE__{
slug: String.t(),
title: String.t()
}