Commit 96bbdbbe authored by Joris's avatar Joris

Fix ouest france ads encoding

parent 3717598b
...@@ -4,6 +4,7 @@ module Service.AdListener ...@@ -4,6 +4,7 @@ module Service.AdListener
import Control.Concurrent (threadDelay) import Control.Concurrent (threadDelay)
import Data.Either (rights) import Data.Either (rights)
import Data.Text.Encoding as T
import qualified Data.Text.IO as T import qualified Data.Text.IO as T
import Prelude hiding (error) import Prelude hiding (error)
...@@ -47,14 +48,30 @@ listenToNewAdsWithViewedURLs conf viewedURLs = do ...@@ -47,14 +48,30 @@ listenToNewAdsWithViewedURLs conf viewedURLs = do
fetchAds :: Conf -> IO [Ad] fetchAds :: Conf -> IO [Ad]
fetchAds conf = do fetchAds conf = do
leboncoinAds <- fmap (concat . map LeboncoinParser.parse . rights) . sequence . map HTTP.get . Conf.leboncoinUrls $ conf leboncoinAds <- getLeboncoinAds conf
ouestFranceAds <- fmap (concat . map OuestFranceParser.parse . rights) . sequence . map HTTP.get . Conf.ouestFranceUrls $ conf ouestFranceAds <- getOuestFranceAds conf
let results = leboncoinAds ++ ouestFranceAds let results = leboncoinAds ++ ouestFranceAds
if null results if null results
then T.putStrLn "Parsed 0 results!" then T.putStrLn "Parsed 0 results!"
else return () else return ()
return results return results
getLeboncoinAds :: Conf -> IO [Ad]
getLeboncoinAds conf =
fmap (concat . map LeboncoinParser.parse . rights)
. sequence
. map (HTTP.get T.decodeLatin1)
. Conf.leboncoinUrls
$ conf
getOuestFranceAds :: Conf -> IO [Ad]
getOuestFranceAds conf =
fmap (concat . map OuestFranceParser.parse . rights)
. sequence
. map (HTTP.get T.decodeUtf8)
. Conf.ouestFranceUrls
$ conf
sendMail :: Conf -> [Ad] -> IO () sendMail :: Conf -> [Ad] -> IO ()
sendMail conf ads = sendMail conf ads =
let (title, plainBody) = Ad.renderAds ads let (title, plainBody) = Ad.renderAds ads
......
...@@ -3,18 +3,17 @@ module Utils.HTTP ...@@ -3,18 +3,17 @@ module Utils.HTTP
) where ) where
import Control.Exception (SomeException, try) import Control.Exception (SomeException, try)
import Data.ByteString (ByteString)
import Data.ByteString.Lazy as BS import qualified Data.ByteString.Lazy as BS
import Data.Text (Text) import Data.Text (Text)
import qualified Data.Text as T import qualified Data.Text as T
import Data.Text.Encoding as T
import Network.HTTP.Conduit import Network.HTTP.Conduit
import Model.URL import Model.URL
import Utils.Either (mapLeft) import Utils.Either (mapLeft)
get :: URL -> IO (Either Text Text) get :: (ByteString -> Text) -> URL -> IO (Either Text Text)
get url = mapLeft (T.pack . show) <$> (try (unsafeGetPage url) :: IO (Either SomeException Text)) get decode url = mapLeft (T.pack . show) <$> (try (unsafeGetPage decode url) :: IO (Either SomeException Text))
unsafeGetPage :: URL -> IO Text unsafeGetPage :: (ByteString -> Text) -> URL -> IO Text
unsafeGetPage url = (T.decodeLatin1 . BS.toStrict) <$> simpleHttp (T.unpack url) unsafeGetPage decode url = (decode . BS.toStrict) <$> simpleHttp (T.unpack url)
...@@ -11,7 +11,6 @@ import Data.List (find, findIndex) ...@@ -11,7 +11,6 @@ import Data.List (find, findIndex)
import Data.Maybe (catMaybes, listToMaybe) import Data.Maybe (catMaybes, listToMaybe)
import Data.Text (Text) import Data.Text (Text)
import qualified Data.Text as T import qualified Data.Text as T
import Text.HTML.TagSoup import Text.HTML.TagSoup
getTagsBefore :: Text -> [Tag Text] -> [Tag Text] getTagsBefore :: Text -> [Tag Text] -> [Tag Text]
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment