Commit 96bbdbbe authored by Joris's avatar Joris

Fix ouest france ads encoding

parent 3717598b
......@@ -4,6 +4,7 @@ module Service.AdListener
import Control.Concurrent (threadDelay)
import Data.Either (rights)
import Data.Text.Encoding as T
import qualified Data.Text.IO as T
import Prelude hiding (error)
......@@ -47,14 +48,30 @@ listenToNewAdsWithViewedURLs conf viewedURLs = do
fetchAds :: Conf -> IO [Ad]
fetchAds conf = do
leboncoinAds <- fmap (concat . map LeboncoinParser.parse . rights) . sequence . map HTTP.get . Conf.leboncoinUrls $ conf
ouestFranceAds <- fmap (concat . map OuestFranceParser.parse . rights) . sequence . map HTTP.get . Conf.ouestFranceUrls $ conf
leboncoinAds <- getLeboncoinAds conf
ouestFranceAds <- getOuestFranceAds conf
let results = leboncoinAds ++ ouestFranceAds
if null results
then T.putStrLn "Parsed 0 results!"
else return ()
return results
getLeboncoinAds :: Conf -> IO [Ad]
getLeboncoinAds conf =
fmap (concat . map LeboncoinParser.parse . rights)
. sequence
. map (HTTP.get T.decodeLatin1)
. Conf.leboncoinUrls
$ conf
getOuestFranceAds :: Conf -> IO [Ad]
getOuestFranceAds conf =
fmap (concat . map OuestFranceParser.parse . rights)
. sequence
. map (HTTP.get T.decodeUtf8)
. Conf.ouestFranceUrls
$ conf
sendMail :: Conf -> [Ad] -> IO ()
sendMail conf ads =
let (title, plainBody) = Ad.renderAds ads
......
......@@ -3,18 +3,17 @@ module Utils.HTTP
) where
import Control.Exception (SomeException, try)
import Data.ByteString.Lazy as BS
import Data.ByteString (ByteString)
import qualified Data.ByteString.Lazy as BS
import Data.Text (Text)
import qualified Data.Text as T
import Data.Text.Encoding as T
import Network.HTTP.Conduit
import Model.URL
import Utils.Either (mapLeft)
get :: URL -> IO (Either Text Text)
get url = mapLeft (T.pack . show) <$> (try (unsafeGetPage url) :: IO (Either SomeException Text))
get :: (ByteString -> Text) -> URL -> IO (Either Text Text)
get decode url = mapLeft (T.pack . show) <$> (try (unsafeGetPage decode url) :: IO (Either SomeException Text))
unsafeGetPage :: URL -> IO Text
unsafeGetPage url = (T.decodeLatin1 . BS.toStrict) <$> simpleHttp (T.unpack url)
unsafeGetPage :: (ByteString -> Text) -> URL -> IO Text
unsafeGetPage decode url = (decode . BS.toStrict) <$> simpleHttp (T.unpack url)
......@@ -11,7 +11,6 @@ import Data.List (find, findIndex)
import Data.Maybe (catMaybes, listToMaybe)
import Data.Text (Text)
import qualified Data.Text as T
import Text.HTML.TagSoup
getTagsBefore :: Text -> [Tag Text] -> [Tag Text]
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment