Commit c0c58753 authored by Erik Aker's avatar Erik Aker
Browse files

Wherein I swoop in and do a bunch of work once a year

parent f6b40b09
*~
*#
.ghci
.DS_STORE
.stack-work/
templates/static
.vscode/
.envrc
dist-newstyle/
\ No newline at end of file
......@@ -3,6 +3,7 @@
.ghci
.DS_STORE
.stack-work/
.DS_Store
templates/static
.vscode/
.envrc
dist-newstyle/
\ No newline at end of file
FROM debian:stretch-slim as builder
ENV LANG C.UTF-8
SHELL ["/bin/bash", "-Eeuxo", "pipefail", "-c"]
RUN apt-get update && \
......@@ -8,23 +9,30 @@ RUN apt-get update && \
libffi-dev \
libgmp-dev \
zlib1g-dev \
gnupg2 \
dirmngr \
curl \
ca-certificates \
tcl \
netbase \
&& curl -sSL "https://get.haskellstack.org/" | sh \
&& rm -rf /var/lib/apt/lists/* \
&& mkdir -p /opt/ekadanta-co/bin
RUN echo 'deb http://downloads.haskell.org/debian stretch main' > /etc/apt/sources.list.d/ghc.list && \
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys BA3CBA3FFE22B574 && \
apt-get update && \
apt-get install -y --no-install-recommends ghc-8.10.2 cabal-install-3.2
ENV PATH /root/.cabal/bin:/root/.local/bin:/opt/cabal/3.2/bin:/opt/ghc/8.10.2/bin:$PATH
WORKDIR /opt/ekadanta-co/
COPY stack.yaml package.yaml ChangeLog.md README.md /opt/ekadanta-co/
COPY ekadanta-co.cabal ChangeLog.md README.md /opt/ekadanta-co/
RUN stack --no-terminal --install-ghc build --only-dependencies
RUN cabal update && cabal build --only-dependencies
COPY . /opt/ekadanta-co
RUN stack install
RUN cabal install
FROM debian:stretch-slim as base_os
......@@ -37,7 +45,7 @@ RUN apt-get update && \
ca-certificates \
netbase
COPY --from=builder /root/.local/bin/ekadanta-co /opt/ekadanta-co/bin/
COPY --from=builder /root/.cabal/bin/ekadanta-co /opt/ekadanta-co/bin/
WORKDIR /opt/ekadanta-co/
......
FROM debian:stretch-slim as builder
# SHELL ["/bin/bash", "-o", "pipefail", "-c"]
ENV LANG C.UTF-8
SHELL ["/bin/bash", "-Eeuxo", "pipefail", "-c"]
RUN apt-get update && \
apt-get install --no-install-recommends -y \
......@@ -8,20 +9,32 @@ RUN apt-get update && \
libffi-dev \
libgmp-dev \
zlib1g-dev \
gnupg2 \
dirmngr \
curl \
ca-certificates \
tcl \
netbase \
&& curl -sSL "https://get.haskellstack.org/" | sh \
&& rm -rf /var/lib/apt/lists/* \
&& mkdir -p /opt/ekadanta-co/bin
RUN echo 'deb http://downloads.haskell.org/debian stretch main' > /etc/apt/sources.list.d/ghc.list && \
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys BA3CBA3FFE22B574 && \
apt-get update && \
apt-get install -y --no-install-recommends ghc-8.10.2 cabal-install-3.2
ENV PATH /root/.cabal/bin:/root/.local/bin:/opt/cabal/3.2/bin:/opt/ghc/8.10.2/bin:$PATH
RUN adduser --disabled-password --gecos "" ekadanta \
&& chown -R ekadanta:ekadanta /opt/ekadanta-co
WORKDIR /opt/ekadanta-co/
COPY stack.yaml package.yaml ChangeLog.md README.md /opt/ekadanta-co/
COPY ekadanta-co.cabal ChangeLog.md README.md /opt/ekadanta-co/
RUN stack --no-terminal --install-ghc build --only-dependencies
RUN cabal update && cabal build ekadanta-test --enable-tests --enable-coverage
COPY . /opt/ekadanta-co
COPY data_files /opt/ekadanta-co/data_files
COPY src /opt/ekadanta-co/src
COPY test /opt/ekadanta-co/test
CMD ["stack", "test"]
CMD ["cabal", "test"]
module Main where
import System.Directory ( doesFileExist )
import System.Environment ( getArgs )
import Data.Aeson (eitherDecodeFileStrict', Value)
import qualified Data.Text as T
import RIO
( FilePath, Either(..), IO, Maybe(..), String,
(<$>), (<>), (>>), (>>=), (||), ($), (.),
map,
mapM, mapM_,
first,
id,
not,
fromRight,
and,
concat,
null,
pure,
undefined,
exitWith,
show,
ExitCode(ExitFailure),
LogLevel(..) )
-- import RIO.Process
import RIO.Time (getCurrentTime)
import Servant.Client ( ClientError )
import System.Envy (decodeEnv, DefConfig(..))
import System.Log.FastLogger
( newStdoutLoggerSet,
defaultBufSize,
pushLogStrLn,
flushLogStr,
LoggerSet,
ToLogStr(..) )
import Site
( SiteConfig(environment, version),
Environment(Local),
LogMessage(..),
jsonRequestLogger )
import Site.Config
import Site.Search ( indexContent )
import Site.Types ( Resource )
logSender :: ToLogStr msg => LoggerSet -> msg -> IO ()
logSender appLogger lgMsg = pushLogStrLn appLogger (toLogStr lgMsg) >> flushLogStr appLogger
jsonFileToResource :: FilePath -> IO (Either String Resource)
jsonFileToResource = eitherDecodeFileStrict'
fpathToLogMsg :: LogMessage -> String -> LogMessage
fpathToLogMsg lgMsg fp = lgMsg {message = "Indexing file: " <> T.pack fp}
mkFailMsg :: LogMessage -> IO LogMessage
mkFailMsg ogLgMsg = do
tstamp <- getCurrentTime
pure LogMessage {
message = "FAIL: "
, timestamp = tstamp
, lversion = lversion ogLgMsg
, application = "ekadanta indexer"
, level = RIO.LevelError
}
indexer :: LogMessage -> SiteConfig -> (LogMessage -> IO a) -> FilePath -> IO (Either T.Text Value)
indexer ogLgMsg config logger fp = do
_ <- logger $ ogLgMsg { message = "Indexing document: " <> T.pack fp }
maybeRes <- jsonFileToResource fp
failMsg <- mkFailMsg ogLgMsg
case maybeRes of
(Right res) -> do
indexRes <- indexContent config Nothing res
case indexRes of
Left fail' -> do
_ <- logger $ failMsg { message = message failMsg <> T.pack fp }
pure $ (Left . T.pack . show ) fail'
Right success -> pure $ Right success
Left err -> do
_ <- logger $ failMsg { message = "Failed to decode file " <> T.pack fp <> " " <> T.pack err}
pure (Left ("Failed to decode file " <> T.pack err))
main :: IO ()
main = do
itemPaths <- getArgs
let itemPathsT = map T.pack itemPaths
itemPathsStr = T.intercalate ", " itemPathsT
conf <- decodeEnv :: IO (Either String SiteConfig)
let config = fromRight (defConfig :: SiteConfig) conf
tstamp <- getCurrentTime
let lgMsg = LogMessage {
message = "Ekadanta App Indexer Script Running with New Doc Path: " <> itemPathsStr
, timestamp = tstamp
, lversion = version config
, application = "ekadanta indexer"
, level = RIO.LevelInfo
}
appLogger <- newStdoutLoggerSet defaultBufSize
-- Get args with item paths and check if they're all present
allPresent <- and <$> mapM doesFileExist itemPaths
if not allPresent || null itemPaths
then do
let lgMsg' = lgMsg { message = "One or more files has failed to exist " <> itemPathsStr }
pushLogStrLn appLogger (toLogStr lgMsg') >> flushLogStr appLogger
exitWith (ExitFailure 1)
else do
let logger = logSender appLogger
mapM_ (indexer lgMsg config logger) itemPaths
......@@ -13,9 +13,8 @@ import RIO
import Servant.Checked.Exceptions.Internal.Servant.API (ErrStatus(toErrStatus))
import Servant
import Servant.Auth.Server
import Servant.Auth.Server.SetCookieOrphan ()
import System.Envy
import System.Log.FastLogger ( newStdoutLoggerSet
import System.Log.FastLogger ( newStdoutLoggerSet
, defaultBufSize
, pushLogStrLn
, flushLogStr
......@@ -25,7 +24,7 @@ import Site
errorMaker :: SomeException -> Wai.Response
errorMaker someErr =
errorMaker someErr =
case (cast someErr :: Maybe AppErrors) of
Just myError -> Wai.responseLBS (toErrStatus myError) [(hContentType, "application/json")] $ encode myError
Nothing -> Wai.responseLBS internalServerError500 [(hContentType, "application/json")] $ "{\"status\": \"failed\", \"error\": \"" <> (encode $ show someErr) <> "\"}"
......@@ -59,7 +58,7 @@ main = do
timeoutSettings = Warp.setTimeout 55 portSettings
settings = Warp.setOnExceptionResponse errorMaker timeoutSettings
jwtCfg = defaultJWTSettings myKey
cookieCfg = if environment config == Local
cookieCfg = if environment config == Local
then defaultCookieSettings{cookieIsSecure=NotSecure
, cookieXsrfSetting = Nothing}
-- "If your web application runs no javascript, disabling XSRF entirely may be required."
......
This diff is collapsed.
......@@ -4,40 +4,39 @@
"number_of_replicas": 2
},
"mappings": {
"content": {
"properties": {
"_resourceType": {
"type": "keyword"
},
"_contentEncoding": {
"type": "keyword"
},
"_featuredImage": {
"type": "text"
},
"_published": {
"type": "boolean"
},
"_body": {
"type": "text"
},
"_title": {
"type": "text"
},
"_lede": {
"type": "text"
},
"_tags": {
"type": "keyword"
},
"_pubdate": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"_pid": {
"enabled": false
}
"properties": {
"_resourceType": {
"type": "keyword"
},
"_contentEncoding": {
"type": "keyword"
},
"_featuredImage": {
"type": "text"
},
"_published": {
"type": "boolean"
},
"_body": {
"type": "text"
},
"_title": {
"type": "text"
},
"_lede": {
"type": "text"
},
"_tags": {
"type": "keyword"
},
"_pubdate": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"_pid": {
"enabled": false
}
}
}
}
\ No newline at end of file
-- This file has been generated from package.yaml by hpack version 0.28.2.
--
-- see: https://github.com/sol/hpack
--
-- hash: fed11044068f28483cf8ef83f69681b5d792ff8a7e9263c891a44d9b1e198ce5
name: ekadanta-co
version: 0.1.0.0
description: Please see the README on GitHub at <https://github.com/erewok/ekadanta-co#readme>
homepage: https://github.com/erewok/ekadanta-co#readme
bug-reports: https://github.com/erewok/ekadanta-co/issues
version: 1.2.0.0
description: Please see the README on Gitlab at <https://gitlab.com/erewok/ekadanta-co-servant/-/blob/master/README.md>
homepage: https://gitlab.com/erewok/ekadanta-co-servant/-/blob/master/README.md
bug-reports: https://gitlab.com/erewok/ekadanta-co-servant/-/issues
author: Erik Aker
maintainer: eraker@gmail.com
copyright: 2018 Erik aker
copyright: 2020 Erik aker
license: BSD3
license-file: LICENSE
build-type: Simple
cabal-version: >= 1.10
cabal-version: >= 2.0
extra-source-files:
ChangeLog.md
README.md
......@@ -24,7 +18,7 @@ data-files:
source-repository head
type: git
location: https://github.com/erewok/ekadanta-co
location: https://gitlab.com/erewok/ekadanta-co-servant/-/tree/master
library
exposed-modules:
......@@ -46,10 +40,11 @@ library
Paths_ekadanta_co
hs-source-dirs:
src
ghc-options: -Wall
default-extensions: DataKinds DeriveGeneric GADTs InstanceSigs LambdaCase MultiWayIf NoImplicitPrelude OverloadedStrings RecordWildCards ScopedTypeVariables TypeOperators UnicodeSyntax
build-depends:
aeson
, base >=4.7 && <5
, base >=4.13 && <5
, bifunctors
, blaze-html
, blaze-markup
......@@ -72,7 +67,7 @@ library
, servant-checked-exceptions
, servant-client
, servant-server
, smtps-gmail
, smtp-mail
, text
, time
, uuid
......@@ -81,6 +76,7 @@ library
, world-peace
default-language: Haskell2010
executable ekadanta-co
main-is: Main.hs
other-modules:
......@@ -88,10 +84,10 @@ executable ekadanta-co
hs-source-dirs:
app
default-extensions: DataKinds DeriveGeneric GADTs InstanceSigs LambdaCase MultiWayIf NoImplicitPrelude OverloadedStrings RecordWildCards ScopedTypeVariables TypeOperators UnicodeSyntax
ghc-options: -threaded -rtsopts -with-rtsopts=-N
ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N
build-depends:
aeson
, base >=4.7 && <5
, base >=4.13 && <5
, bifunctors
, blaze-html
, blaze-markup
......@@ -115,7 +111,7 @@ executable ekadanta-co
, servant-checked-exceptions
, servant-client
, servant-server
, smtps-gmail
, smtp-mail
, text
, time
, uuid
......@@ -125,12 +121,12 @@ executable ekadanta-co
, world-peace
default-language: Haskell2010
test-suite ekadanta-co-test
test-suite ekadanta-test
type: exitcode-stdio-1.0
main-is: Spec.hs
other-modules:
PublicResourcesSpec
Paths_ekadanta_co
Site.PublicResourcesSpec
hs-source-dirs:
test
default-extensions: DataKinds DeriveGeneric GADTs InstanceSigs LambdaCase MultiWayIf NoImplicitPrelude OverloadedStrings RecordWildCards ScopedTypeVariables TypeOperators UnicodeSyntax
......@@ -138,40 +134,51 @@ test-suite ekadanta-co-test
build-depends:
QuickCheck
, aeson
, base >=4.7 && <5
, bifunctors
, blaze-html
, blaze-markup
, bytestring
, cmark
, cmark-gfm
, data-default
, base >=4.13 && <5
, deepseq
, ekadanta-co
, envy
, fast-logger
, hspec
, hspec-discover
, hspec-wai
, hspec-wai-json
, http-api-data
, http-client
, http-types
, lens
, lens-aeson
, mime-mail
, rio
, servant
, servant-auth-server
, servant-blaze
, servant-checked-exceptions
, servant-client
, servant-server
, smtps-gmail
, text
, time
, uuid
, wai
, wai-extra
, warp
, world-peace
default-language: Haskell2010
executable ekadanta-indexer
main-is: Indexer.hs
other-modules:
Paths_ekadanta_co
hs-source-dirs:
app
default-extensions: DataKinds DeriveGeneric GADTs InstanceSigs LambdaCase MultiWayIf NoImplicitPrelude OverloadedStrings RecordWildCards ScopedTypeVariables TypeOperators UnicodeSyntax
ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N
build-depends:
aeson
, base >=4.13 && <5
, bifunctors
, bytestring
, data-default
, directory
, ekadanta-co
, envy
, fast-logger
, rio
, servant-client
, text
default-language: Haskell2010
\ No newline at end of file
name: ekadanta-co
version: 0.1.0.0
github: "erewok/ekadanta-co"
license: BSD3
author: "Erik Aker"
maintainer: "eraker@gmail.com"
copyright: "2018 Erik aker"
extra-source-files:
- README.md
- ChangeLog.md
# Metadata used when publishing your package
# synopsis: Short description of your package
# category: Web
# To avoid duplicated efforts in documentation and dealing with the
# complications of embedding Haddock markup inside cabal files, it is
# common to point users to the README.md file.
description: Please see the README on GitHub at <https://github.com/erewok/ekadanta-co#readme>
dependencies:
- base >= 4.7 && < 5
- aeson
- bifunctors
- blaze-html
- blaze-markup
- bytestring
- cmark
- cmark-gfm
- data-default
- envy
- fast-logger
- http-api-data
- http-client
- http-types
- lens
- lens-aeson
- mime-mail
- rio
- servant
- servant-auth-server
- servant-blaze
- servant-checked-exceptions
- servant-client
- servant-server
- smtps-gmail
- text
- time
- uuid
- wai
- wai-extra
- world-peace
default-extensions:
- DataKinds
- DeriveGeneric
- GADTs
- InstanceSigs
- LambdaCase
- MultiWayIf
- NoImplicitPrelude
- OverloadedStrings
- RecordWildCards
- ScopedTypeVariables
- TypeOperators
- UnicodeSyntax
data-files:
- data_files/es_ekadanta_content_mapping.json
library:
source-dirs: src
executables:
ekadanta-co:
main: Main.hs
source-dirs: app
ghc-options:
- -threaded
- -rtsopts
- -with-rtsopts=-N
dependencies:
- ekadanta-co
- http-types
- wai
- wai-extra
- warp
tests:
ekadanta-co-test:
main: Spec.hs
source-dirs: test
ghc-options:
- -threaded
- -rtsopts
- -with-rtsopts=-N
dependencies:
- ekadanta-co
- deepseq
- hspec
- hspec-wai
- hspec-wai-json
- QuickCheck
- wai
- wai-extra
- warp
- aeson
- lens-aeson
......@@ -15,7 +15,6 @@ import RIO hiding ( Handler )
import qualified RIO.HashMap as HM
import Servant
import Servant.Auth.Server
import Servant.Auth.Server.SetCookieOrphan ()
import Site.Config as X
import Site.Exceptions as X
......@@ -26,8 +25,8 @@ import Site.Types as X
type SiteWideApi auths =
PublicApi
type SiteWideApi auths =
PublicApi
:<|> AdminAndLogin auths
:<|> "health" :> Get '[JSON] Value