Commit cf70a8a9 authored by Dave Morriss's avatar Dave Morriss

Changes for alternative audio types

Makefile: builds all Asciidoc documentation
README.pdf: regenerated from Asciidoc version
cronjob: script to be called from cron to tidy logs
get_album: can handle downloading FLAC or OGG
parse_magnatune_{flac,ogg}.xsl: needed for parsing out FLAC/OGG details
make_album_summary, report_albumsku, update_albums: general tidying
parent fa81148d
#
# Makefile to build all documentation
# Makefile to build all documentation.
#
# Note we use 'asciidoctor' rather than 'asciidoc' since the latter seems to
# be obsolete. Note also that the PDF conversion relies on the 'fop' package.
#
SHELL = /bin/sh
TEXTS = README get_album.txt report_albumsku.txt update_albums.txt
HTMLS = README.html
HTMLS += $(TEXTS:%.txt=%.html)
TEXTS = README.txt get_album.txt report_albumsku.txt update_albums.txt
HTMLS = $(TEXTS:%.txt=%.html)
PDFS = $(HTMLS:%.html=%.pdf)
all : $(HTMLS) $(PDFS)
#
# Special rules for these two since they are non-standard
# Generic rules for building HTML and PDF
#
README.html : README
asciidoc -b html README
README.pdf : README
a2x -f pdf --fop README
#get_album.html : get_album.txt
# asciidoc -b html get_album.txt
#get_album.pdf : get_album.txt
# a2x -f pdf --fop get_album.txt
#report_albumsku.html : report_albumsku.txt
# asciidoc -b html report_albumsku.txt
#report_albumsku.pdf : report_albumsku.txt
# a2x -f pdf --fop report_albumsku.txt
#update_albums.html : update_albums.txt
# asciidoc -b html update_albums.txt
#update_albums.pdf : update_albums.txt
# a2x -f pdf --fop update_albums.txt
%.html : %.txt
asciidoc -b html -o $@ $<
asciidoctor -b html -o $@ $<
%.pdf : %.txt
a2x -f pdf --fop $<
No preview for this file type
#!/bin/bash -
#===============================================================================
#
# FILE: cronjob
#
# USAGE: ./cronjob
#
# DESCRIPTION: Script to be run from cron for managing ~/MusicDownloads
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# LICENSE: Copyright (c) year 2013, Dave Morriss
# CREATED: 2013-02-27 09:50:42
# REVISION: 2016-01-11 14:55:10
#
#===============================================================================
set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
BASEDIR="$HOME/MusicDownloads"
LOGDIR="$BASEDIR/logs/"
MONTHS4=$(( 28 * 4 ))
MONTHS9=$(( 28 * 9 ))
#
# Tidy log files
#
find $LOGDIR -type f -mtime +${MONTHS9} -printf "Deleted %f\n" -exec rm -f {} \;
exit
# vim: syntax=sh:ts=8:sw=4:ai:et:tw=78:fo=tcrqn21
#!/bin/bash -
#!/bin/bash -
#===============================================================================
#
# FILE: get_album
#
# USAGE: ./get_album
# USAGE: ./get_album [-h] [-c] magnatune-code
#
# DESCRIPTION: Script to buy and download a Magnatune album and record the
# purchase. Also downloads the cover in medium and large sizes
......@@ -14,10 +14,10 @@
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# LICENSE: Copyright (c) year 2012, 2013, 2014 Dave Morriss
# VERSION: 0.1.1
# LICENSE: Copyright (c) year 2012, 2013, 2014, 2015 Dave Morriss
# VERSION: 0.1.2
# CREATED: 2012-10-14 22:02:32
# REVISION: 2014-07-12 23:53:06
# REVISION: 2016-01-11 11:11:48
#
#===============================================================================
......@@ -26,53 +26,6 @@ set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/} # Basename of script
DIR=${0%/*} # Dirname of script
#-------------------------------------------------------------------------------
# Directories:
# $BASEDIR--+
# +--Magnatune_Data Data files here
# +--Magnatune_Downloads Music downloads go here
# +--magnatune-downloader Scripts
# +--logs Download logs etc
#-------------------------------------------------------------------------------
#
# Define the places where the scripts and data files are to be stored
#
# Root directory
BASEDIR="$HOME/MusicDownloads"
# Data files go here
DATADIR="$BASEDIR/Magnatune_Data"
# Scripts live here
SCRIPTDIR="$BASEDIR/magnatune-downloader"
# Where the downloads go
DOWNLOADDIR="$BASEDIR/Magnatune_Downloads"
#
# Files that will be created/updated
#
SUMMARY="$DATADIR/all_albums"
PURCHASES="$SCRIPTDIR/purchased"
#
# XSL stuff
#
XSLT="$SCRIPTDIR/parse_magnatune.xsl"
#
# Where the Magnatune username, password and id are found. The RCFILE must
# contain three lines, order not significant:
# user="magnatune_username"
# pass='password'
# id=magnatune_id
#
# The 'id' line can be omitted if there is none, in which case the 'user'
# value is used by the script.
# Use either type of quote depending on whether there are spaces in the
# strings.
#
RCFILE="$HOME/.magnatunerc"
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#=== FUNCTION ================================================================
# NAME: check_download
# DESCRIPTION: Checks a download worked
......@@ -104,7 +57,7 @@ function cleanup_temp {
[ -e $tmp ] && rm --force $tmp
done
exit 0
}
}
#=== FUNCTION ================================================================
# NAME: yes_no
......@@ -153,7 +106,7 @@ _usage () {
Usage: ${SCRIPT} [-h] [-c] magnatune-code
Downloads the ogg file and/or the cover for a Magnatune album
Downloads the audio file and/or the cover for a Magnatune album
Options:
-c This option, if set, causes just the cover to be
......@@ -174,10 +127,33 @@ endusage
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#-------------------------------------------------------------------------------
# Directories:
# $BASEDIR--+
# +--Magnatune_Data Data files here
# +--Magnatune_Downloads Music downloads go here
# +--magnatune-downloader Scripts
# +--logs Download logs etc (unmanaged)
#-------------------------------------------------------------------------------
#
# Define the places where the scripts and data files are to be stored
#
BASEDIR="$HOME/MusicDownloads"
# Data files go here
DATADIR="$BASEDIR/Magnatune_Data"
# Scripts live here
SCRIPTDIR="$BASEDIR/magnatune-downloader"
# Where the downloads go
DOWNLOADDIR="$BASEDIR/Magnatune_Downloads"
SUMMARY="$DATADIR/all_albums"
PURCHASES="$SCRIPTDIR/purchased"
RCFILE="$HOME/.magnatunerc"
#
# Sanity checks
#
[ -e $XSLT ] || { echo "$XSLT not found"; exit 1; }
[ -e $RCFILE ] || { echo "$RCFILE not found"; exit 1; }
[ -e $SUMMARY ] || { echo "$SUMMARY not found"; exit 1; }
[ -e $PURCHASES ] || { echo "$PURCHASES not found"; exit 1; }
......@@ -194,6 +170,15 @@ USER="${ARGS[user]:?No username in $RCFILE}"
PASS="${ARGS[pass]:?No password in $RCFILE}"
MYID="${ARGS[id]:-$USER}"
#
# The audio type (we cater for FLAC and OGG only at present) decides which
# XSLT file we'll use. We check it exists.
# TODO: Think about making the audio type an option
#
AUDIO=flac
printf -v XSLT "$SCRIPTDIR/parse_magnatune_%s.xsl" $AUDIO
[ -e $XSLT ] || { echo "$XSLT not found"; exit 1; }
#
# Process options
#
......@@ -276,16 +261,17 @@ if [[ $DLXML -eq 1 ]]; then
#
# Download the XML file containing the download URLs
# TODO check for download errors?
#
echo "Downloading XML for code '$CODE'"
curl $URL -o "$XML"
#---------------------------------------------------------------------------
#
# The downloaded XML may not be well-formed and may need to be mended.
# The downloaded XML may not be well-formed and may need to be mended
# The following code can detect some problems and correct them. It's not
# the right way to fix XML but you may want to risk it.
# You have been warned :-)
# You have been warned :-) Un-comment the following at your peril!
#
# if grep -i "<br>" "$XML" > /dev/null; then
# sed -i.bck -e 's:<br>:<br/>:' "$XML"
......@@ -299,7 +285,7 @@ if [[ $DLXML -eq 1 ]]; then
fi
#
# The XSLT stage. Parse out the URLs for the cover and the OGG file
# The XSLT stage. Parse out the URLs for the cover and the audio file.
#
# Bad XML at this stage will cause the script to abort with an error. It will
# offer the option of deleting the XML file, assuming it'll be fixed by
......@@ -321,7 +307,12 @@ if [[ ${#TMP[@]} -ne 2 ]]; then
exit
fi
COVER=${TMP[0]}
OGGFILE=${TMP[1]}
AUDIOFILE=${TMP[1]}
# echo "XML=$XML"
# echo "COVER=$COVER"
# echo "AUDIOFILE=$AUDIOFILE"
# exit
#
# Also make URLS for the full cover and the artwork
......@@ -330,15 +321,15 @@ FULLCOVER="${COVER%/*}/cover.jpg"
PDF="${COVER%/*}/artwork.pdf"
#
# Download the OGG, with authentication
# Download the audio, with authentication
#
if [[ $COVERONLY -eq 0 ]]; then
echo "Downloading OGG file"
URL="${OGGFILE:0:7}${USER}:${PASS}@${OGGFILE:7}"
OGGPATH="$DOWNLOADDIR/${OGGFILE##*/}"
curl $URL -o "$OGGPATH"
echo "Downloading audio file"
URL="${AUDIOFILE:0:7}${USER}:${PASS}@${AUDIOFILE:7}"
AUDIOPATH="$DOWNLOADDIR/${AUDIOFILE##*/}"
curl $URL -o "$AUDIOPATH"
else
echo "OGG file not downloaded"
echo "Audio file not downloaded"
fi
#
......@@ -357,7 +348,7 @@ PDFPATH="$DOWNLOADDIR/${CODE}_${PDF##*/}"
# report what went wrong.
#
if [[ $COVERONLY -eq 1 ]]; then
check_download "$COVER" "$COVERPATH"
check_download "$COVER" "$COVERPATH"
check_download "$FULLCOVER" "$FULLCOVERPATH"
check_download "$PDF" "$PDFPATH"
......@@ -365,12 +356,12 @@ if [[ $COVERONLY -eq 1 ]]; then
echo "Note: This album has not been recorded as purchased"
fi
else
check_download "$OGGFILE" "$OGGPATH"
check_download "$AUDIOFILE" "$AUDIOPATH"
check_download "$COVER" "$COVERPATH"
check_download "$FULLCOVER" "$FULLCOVERPATH"
check_download "$PDF" "$PDFPATH"
if [[ -e "$OGGPATH" && -e "$COVERPATH" ]]; then
if [[ -e "$AUDIOPATH" && -e "$COVERPATH" ]]; then
echo "$CODE" >> $PURCHASES
#
......
#!/bin/bash -
#!/bin/bash -
#===============================================================================
#
# FILE: make_album_summary
#
# USAGE: ./make_album_summary
# USAGE: ./make_album_summary
#
# DESCRIPTION: Make the file 'all_albums' from the Magnatune album data
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: This script is largely redundant snce the funcitonality is
# included in 'update_albums'
# NOTES: This script is largely redundant since the functionality is
# included in 'update_albums'. UPDATE: keep around since it can
# recover after a failure of 'update_albums'.
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# LICENSE: Copyright (c) year 2012, Dave Morriss
# CREATED: 24/10/2012 15:28:04
# REVISION: 25/11/2012 17:21:55
# CREATED: 2012-10-24 15:28:04
# REVISION: 2016-01-11 14:24:14
#
#===============================================================================
......@@ -23,6 +24,23 @@ set -o nounset # Treat unset variables as an error
SCRIPT=${0##*/}
#=== FUNCTION ================================================================
# NAME: cleanup_temp
# DESCRIPTION: Cleanup temporary files in case of a keyboard interrupt
# (SIGINT) or a termination signal (SIGTERM) and at script
# exit
# PARAMETERS: * - names of temporary files to delete
# RETURNS: Nothing
#===============================================================================
function cleanup_temp {
for tmp in $*; do
[ -e $tmp ] && rm --force $tmp
done
exit 0
}
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#-------------------------------------------------------------------------------
# Directories:
# $BASEDIR--+
......@@ -32,8 +50,6 @@ SCRIPT=${0##*/}
# +--logs Download logs etc (unmanaged)
#-------------------------------------------------------------------------------
#
# Define the place where the scripts and data files are to be stored
#
BASEDIR="$HOME/MusicDownloads"
# Data files go here
DATADIR="$BASEDIR/Magnatune_Data"
......@@ -48,10 +64,40 @@ ALBUMINFO="$DATADIR/album_info_xml"
SUMMARY="$DATADIR/all_albums"
#
# Process the XML
# Sanity checks
#
xsltproc $XSLT $ALBUMINFO > $SUMMARY
[ -e $XSLT ] || { echo "$XSLT not found"; exit 1; }
[ -e $ALBUMINFO ] || { echo "$ALBUMINFO not found"; exit 1; }
#
# Make temporary files and set traps to delete them
#
TMP1=$(mktemp) || { echo "$SCRIPT: creation of temporary file failed!"; exit 1; }
trap "cleanup_temp $TMP1" SIGHUP SIGINT SIGPIPE SIGTERM EXIT
#
# Process the XML, trapping error messages
#
xsltproc $XSLT $ALBUMINFO > $SUMMARY 2>$TMP1
res=$?
#
# If we received an error return report it and do some cleanup
#
if [[ $res -ne 0 ]]; then
echo "Errors processing $ALBUMINFO"
echo "Last few lines from error report:"
echo "----"
tail $TMP1
echo "----"
echo "Invalid output file $SUMMARY deleted"
rm -f $SUMMARY
exit $res
fi
#
# All went well
#
echo "Created '${SUMMARY##*/}' - $(grep ^Album: $SUMMARY | wc -l) albums"
exit
......
<?xml version="1.0" encoding="ISO-8859-1"?>
<xsl:stylesheet version="1.0"
xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
<xsl:output method="text"/>
<xsl:template match="/">
<xsl:apply-templates select="RESULT/meta[@property='og:image']"/>
<xsl:apply-templates select="RESULT"/>
</xsl:template>
<xsl:template match="meta">
<xsl:value-of select="./@content"/><xsl:text>&#10;</xsl:text>
</xsl:template>
<xsl:template match="RESULT">
<xsl:value-of select="URL_FLACZIP"/><xsl:text>&#10;</xsl:text>
</xsl:template>
</xsl:stylesheet>
<?xml version="1.0" encoding="ISO-8859-1"?>
<xsl:stylesheet version="1.0"
xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
<xsl:output method="text"/>
<xsl:template match="/">
<xsl:apply-templates select="RESULT/meta[@property='og:image']"/>
<xsl:apply-templates select="RESULT"/>
</xsl:template>
<xsl:template match="meta">
<xsl:value-of select="./@content"/><xsl:text>&#10;</xsl:text>
</xsl:template>
<xsl:template match="RESULT">
<xsl:value-of select="URL_OGGZIP"/><xsl:text>&#10;</xsl:text>
</xsl:template>
</xsl:stylesheet>
#!/bin/bash -
#!/bin/bash -
#===============================================================================
#
# FILE: report_albumsku
#
# USAGE: ./report_albumsku
# USAGE: ./report_albumsku albumcode
#
# DESCRIPTION: Report album details from the Magnatune catalogue by album
# code (which Magnatune calls the 'sku')
......@@ -14,8 +14,8 @@
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# LICENSE: Copyright (c) year 2012, Dave Morriss
# CREATED: 24/10/2012 21:15:54
# REVISION: 25/11/2012 17:30:41
# CREATED: 2012-10-24 21:15:54
# REVISION: 2016-01-11 14:15:27
#
#===============================================================================
......@@ -50,12 +50,12 @@ SUMMARY="$DATADIR/all_albums"
#
# Sanity checks
#
[ -e $BASEDIR ] || { echo "$BASEDIR not found"; exit; }
[ -e $DATADIR ] || { echo "$DATADIR not found"; exit; }
[ -e $SCRIPTDIR ] || { echo "$SCRIPTDIR not found"; exit; }
[ -e $XSLT ] || { echo "$XSLT not found"; exit; }
[ -e $ALBUMINFO ] || { echo "$ALBUMINFO not found"; exit; }
[ -e $SUMMARY ] || { echo "$SUMMARY not found"; exit; }
[ -e $BASEDIR ] || { echo "$BASEDIR not found"; exit 1; }
[ -e $DATADIR ] || { echo "$DATADIR not found"; exit 1; }
[ -e $SCRIPTDIR ] || { echo "$SCRIPTDIR not found"; exit 1; }
[ -e $XSLT ] || { echo "$XSLT not found"; exit 1; }
[ -e $ALBUMINFO ] || { echo "$ALBUMINFO not found"; exit 1; }
[ -e $SUMMARY ] || { echo "$SUMMARY not found"; exit 1; }
#
# Check argument count
......
......@@ -12,9 +12,9 @@
# BUGS: ---
# NOTES: ---
# AUTHOR: Dave Morriss (djm), Dave.Morriss@gmail.com
# LICENSE: Copyright (c) year 2012, 2013, Dave Morriss
# CREATED: 21/10/2012 11:36:56
# REVISION: 27/02/2013 13:41:12
# LICENSE: Copyright (c) year 2012, 2014 Dave Morriss
# CREATED: 2012-10-21 11:36:56
# REVISION: 2016-01-11 14:24:06
#
#===============================================================================
......@@ -34,13 +34,12 @@ DIR=${0%%/*} # Dirname of script
#
# Define the place where the scripts and data files are to be stored
#
# Root directory
BASEDIR="$HOME/MusicDownloads"
# Data files go here
DATADIR="$BASEDIR/Magnatune_Data"
# Scripts live here
SCRIPTDIR="$BASEDIR/magnatune-downloader"
# We save logs here
# Where logs go (unmanaged, so careful)
LOGDIR="$BASEDIR/logs"
#
......@@ -61,11 +60,11 @@ SUMMARY="$DATADIR/all_albums"
#
# Sanity checks
#
[ -e $BASEDIR ] || { echo "$BASEDIR not found"; exit 1; }
[ -e $DATADIR ] || { echo "$DATADIR not found"; exit 1; }
[ -e $SCRIPTDIR ] || { echo "$SCRIPTDIR not found"; exit 1; }
[ -e $LOGDIR ] || { echo "$LOGDIR not found"; exit 1; }
[ -e $XSLT ] || { echo "$XSLT not found"; exit 1; }
[ -e $BASEDIR ] || { echo "$SCRIPT: $BASEDIR not found"; exit 1; }
[ -e $DATADIR ] || { echo "$SCRIPT: $DATADIR not found"; exit 1; }
[ -e $SCRIPTDIR ] || { echo "$SCRIPT: $SCRIPTDIR not found"; exit 1; }
[ -e $LOGDIR ] || { echo "$SCRIPT: $LOGDIR not found"; exit 1; }
[ -e $XSLT ] || { echo "$SCRIPT: $XSLT not found"; exit 1; }
#
# Have there been any changes since the last download?
......@@ -97,12 +96,12 @@ if [[ "$CHANGED" -eq 1 ]]; then
# Now make the summary file
#
xsltproc $XSLT $ALBUMINFO > $SUMMARY
echo "Created '${SUMMARY##*/}' - $(grep ^Album: $SUMMARY | wc -l) albums"
echo "$SCRIPT: Created '${SUMMARY##*/}' - $(grep ^Album: $SUMMARY | wc -l) albums"
else
echo "Download failure? $ALBUMFILE not found"
echo "$SCRIPT: Download failure? $ALBUMFILE not found"
fi
else
echo "No changes"
echo "$SCRIPT: No changes"
fi
exit
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment