path: root/office/texlive/prep/texmf_get.sh

#!/bin/bash

# texmf_get.sh (c) 2016 Johannes Schoepfer, slackbuilds[at]schoepfer[dot]info
# All rights reserved.
#
# Redistribution and use of this script, with or without modification, is
# permitted provided that the following conditions are met:
#
# 1. Redistributions of this script must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
#  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ''AS IS'' AND ANY EXPRESS OR IMPLIED
#  WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
#  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO
#  EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
#  OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
#  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
#  OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
#  ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
#  V 0.6
#
#  Prepare xz-compressed tarballs of texlive-texmf-trees based on texlive.tlpdb
#  This script takes care of dependencies(as far as these are present in texlive.tlpdb) of collections and packages,
#  and that every texlive-package is included only once.
#
#  Policy from texlive-netarchive: Every package is included as dependency in exactly one collection. A package may have dependecies on other packages from any collection.

# packages source: http://mirror.ctan.org/systems/texlive/tlnet/archive/

#set -e
MAJORVERSION=2016
# release mirror
mirror="http://mirror.ctan.org/systems/texlive/tlnet/"
# pre-test mirror 2016
# mirror="http://ftp.cstug.cz/pub/tex/local/tlpretest/"
LANG=C
TMP=$PWD/tmp
output=$TMP/texlive.packages
output_doc=$TMP/texlive.doc.packages
errorlog=$TMP/error.log
texmf=$TMP/texmf
db=texlive.tlpdb
tmpfile=$(mktemp)
collections_done=$TMP/done
collections_tobedone=$TMP/tobedone

packages () {
# fonts-package first to make sure that big fonts like cm-super are not included elsewhere as dependency
  PACKAGES="
    cm-super
    cbfonts
    sanskrit-t1
    cmcyr
    cs
    uhc
    fonts-tlwg
    ethiop-t1
    ipaex
    wadalab
    fandol
    arphic
    nanumtype1" \
  texmfget fonts

# The base
  PACKAGES="
    collection-basic
    collection-latex
    collection-genericrecommended
    collection-latexrecommended
    collection-xetex
    collection-metapost
    collection-plainextra
    collection-fontutils
    collection-genericextra
    collection-formatsextra
    collection-htmlxml
    collection-luatex
    collection-fontsrecommended
    collection-mathextra
    collection-humanities
    collection-context
    lh
    yfonts
    doublestroke
    was
    xypic
    xindy
    barcodes
    qrcode
    lastpage
    datetime2
    texdoc
    appendix
    changebar
    footmisc
    multirow
    overpic
    stmaryrd
    subfigure
    titlesec
    siunitx
    combelow
    csplain
    csquotes
    etoolbox
    etextools
    glossaries
    imakeidx
    idxlayout
    bidi
    filecontents
    biblatex
    biber.x86_64-linux
    biber.i386-linux
    collection-langeuropean
    collection-langenglish
    collection-langfrench
    collection-langgerman
    collection-langitalian
    collection-langpolish
    collection-langportuguese
    collection-langspanish
    collection-langgreek
    collection-langafrican
    hyphen-czech
    hyphen-slovak
    hyphen-indic
    hyphen-sanskrit
    hyphen-armenian
    hyphen-afrikaans
    hyphen-esperanto
    hyphen-bulgarian
    hyphen-churchslavonic
    hyphen-mongolian
    hyphen-russian
    hyphen-serbian
    hyphen-ukrainian
    hyphen-catalan
    hyphen-galician
    hyphen-chinese
    hyphen-coptic
    hyphen-georgian
    hyphen-indonesian
    hyphen-interlingua
    hyphen-thai
    hyphen-turkmen
    hyphen-arabic
    hyphen-farsi" \
  texmfget base

# Call "fonts"-tarball again to add remaining fonts
PACKAGES="collection-fontsextra" texmfget fonts

# Put all remaining stuff in the "extra" tarball
  PACKAGES="
    collection-latexextra
    collection-pictures
    collection-games
    collection-publishers
    collection-bibtexextra
    collection-binextra
    collection-science
    collection-omega
    collection-music
    collection-langother
    collection-pstricks
    collection-langcyrillic
    collection-langczechslovak
    collection-langindic
    collection-langjapanese
    collection-langkorean
    collection-langarabic
    collection-langchinese
    collection-langcjk" \
  texmfget extra

# The docs-tarball - very big (about 1300 MB)
  texmfget docs

  if [ "$TARBALL" != docs ]
  then
	echo "Packages-list: $output.meta.$TARBALL"
	while read collection
	do
		grep -w "$collection" $collections_done &> /dev/null
		[ $? != 0 ] && echo "WARNING: $collection was not processed, please edit packages-function."
	done < $TMP/allcollections
  fi

# Following aren't supported
#NAME=context PACKAGES="collection-context" ./texmf_get.sh
#NAME=texworks PACKAGES="collection-texworks" ./texmf_get.sh
#NAME=wintools PACKAGES="collection-wintools" ./texmf_get.sh

# For the records:
#
# base-tarball:
# hyphen-packages are for "fmtutil-sys -all" to proceed without errors
#
# for building dblatex:
#   appendix
#   changebar
#   footmisc
#   multirow
#   overpic
#   stmaryrd
#   subfigure
#   titlesec

# for math masters thesis
#   doublestroke
#   was

# decided these are commonly useful and not too big, or or small to just have it for wider support of the base-package
#   csplain
#   yfonts

# to make biber functional
#   biblatex

}

# ==== Nothing to edit beyond this line (hopefully) ====

usage () {
	echo "Prepare texmf trees based on collections and packages and their dependencies."
	echo "./texmf_get.sh [base|docs|extra|fonts]"
	exit
}


package_meta () {
	echo "collection/package $collection"
	# collection start linenumer
	start_n="$(grep -n ^"name ${collection}$" $TMP/$db | cut -d':' -f1)"
	[ -z "$start_n" ] && echo "$collection was not found in $TMP/$db, bye." && exit 1

	# find end of package/collection
	for emptyline in $emptylines
	do
		if [ "$emptyline" -gt "$start_n" ]
		then
			end_n=$emptyline
			break
		fi
	done

	sed "${start_n},${end_n}!d" $TMP/$db > $tmpfile
}

package_list () {
# Only do something if $collection wasn't already done before
while [ -s $collections_tobedone ]
do
	collection=$(tail -n1 $collections_tobedone)
	# continue with next collection if collection was already done
	if [ -s "$collections_done" ]
	then
		grep -w "^${collection}$" $collections_done &> /dev/null
		if [ $? = 0 ]
		then
			sed -i "/^$collection$/d" $collections_tobedone
			continue
		fi
	fi

	if [ -s "$texmf/$collection.meta" ]
	then
		cp $texmf/$collection.meta $tmpfile
	else
		package_meta
	fi

	# ignore dependend collections generally, as this adds far too much and therefore reduces controll over what packages to be added
	sed -i "/^depend collection/d" $tmpfile
	# If $collection is a singel package, add it here
	if [ -n "$(head -n1 $tmpfile | fgrep -v "name collection" )" ]
	then
		packagelist="$TMP/run.tlpkg"
		if [ -z "$(grep -w "^${collection}$" $packagelist)" ]
		then
			sed -i "/^$collection$/d" $collections_tobedone
			echo "$collection" >> $collections_done
			continue
		fi
		echo "$collection" >> $output
	fi
	# add dependend packages
	grep ^"depend " $tmpfile | grep -v "ARCH$" | cut -d' ' -f2- >> $collections_tobedone
	echo "$collection" >> $collections_done
	sed -i "/^${collection}$/d" $collections_tobedone
done

}

untar () {
	# download packages, if not already available. Not for all packages a corresponding .doc package exists
	rm $1.meta
	while read package
	do
		# untar all packages, check for relocation, "relocate 1" -> untar in texmf-dist
		if [ "$flavour" = ".doc" ]
		then
			sha512="$(grep ^doccontainerchecksum $texmf/$package.meta | cut -d' ' -f2 )"
		else
			sha512="$(grep ^containerchecksum $texmf/$package.meta | cut -d' ' -f2 )"
		fi
		# try two times if package isn't present, with -t1 to get another mirror the second time
		[ ! -s ${package}${flavour}.tar.xz ] && wget -t1 -c ${mirror}archive/${package}${flavour}.tar.xz
		[ ! -s ${package}${flavour}.tar.xz ] && wget -t1 -c ${mirror}archive/${package}${flavour}.tar.xz
		[ ! -s ${package}${flavour}.tar.xz ] && echo "Downloading ${package}${flavour}.tar.xz did not work, writing to $errorlog" && echo "Error downloading ${package}${flavour}.tar.xz" >> $errorlog && exit 1
		# check sha512, give three tries for downloading aggain(diffrent mirrors are used automatically)
		for tillthree in 1 2 3
		do
			if [ "$(sha512sum ${package}${flavour}.tar.xz | cut -d' ' -f1 )" != "$sha512" ]
			then
				# Download (hopefully) newer file
				rm ${package}${flavour}.tar.xz
				wget -t1 -c ${mirror}archive/${package}${flavour}.tar.xz
			else
				break
			fi
		done
		# check sha512 again, exit if it fails
		if [ "$(sha512sum ${package}${flavour}.tar.xz | cut -d' ' -f1 )" != "$sha512" ]
		then
			echo "sha512sum $(sha512sum ${package}${flavour}.tar.xz | cut -d' ' -f1 ) of"
			echo "${package}${flavour}.tar.xz doesn't match with $TMP/$db"
			echo "sha512sum $sha512"
			echo "Delete $TMP/${db}* and try again."
			exit 1
		fi

		# exclude the tlpkg-stuff, TLUtils.pm(needed tu run texlive) comes from source installation
		grep -w ^"relocated 1" $texmf/$package.meta &>/dev/null
		if [ $? = 0 ]
		then
			tar vxf ${package}${flavour}.tar.xz --exclude tlpkg -C texmf-dist || exit 1
		else
			tar vxf ${package}${flavour}.tar.xz --exclude tlpkg  || exit 1
		fi
		# in case a binary package was decompressed, put it in texmf-dist
		[ -d bin ] && cp -a bin texmf-dist && rm -rf bin
		if [ "$flavour" = ".doc" ]
		then
			size=$(( $(grep ^doccontainersize $texmf/$package.meta | cut -d' ' -f2 ) / 1024 ))
		else
			size=$(( $(grep ^containersize $texmf/$package.meta | cut -d' ' -f2 ) / 1024 ))
		fi
		shortdesc="$(grep ^shortdesc $texmf/$package.meta | cut -d' ' -f2- )"
		echo "$size Kb, $package: $shortdesc" >> $1.meta
	done < $1
	# copy packages-list to texmf-dist, so included packages are known in later installation
	sort -n $1.meta > TMPFILE
	mv TMPFILE $1.meta
	cp $1.meta texmf-dist/

}

remove_cruft () {
	# Remove m$-stuff, ConTeXt single-user-system stuff, KOMA-Script sources and pdf-manpages
	rm -rf texmf-dist/source
	rm -rf texmf-dist/scripts/context/stubs/source/
	find texmf-dist/ -type d -name 'win32'		-exec rm -rf {} +
	find texmf-dist/ -type d -name 'win64'		-exec rm -rf {} +
	find texmf-dist/ -type d -name 'mswin'		-exec rm -rf {} +
	find texmf-dist/ -type d -name 'win'		-exec rm -rf {} +
	find texmf-dist/ -type d -name 'setup'		-exec rm -rf {} +
	find texmf-dist/ -type d -name 'install'	-exec rm -rf {} +
	find texmf-dist/ -type f -name '*.bat'		-delete
	find texmf-dist/ -type f -name '*win32*'	-delete
	find texmf-dist/ -type f -name 'winansi*'	-delete
	find texmf-dist/ -type f -name '*-man.pdf'	-delete
	# remove zero-length files, as these appear e.g. in hyph-utf8 tex-package.
	find . -type f -size 0c -delete
}

texmfget () {

NAME="$1"
# remove outputfile if already present
[ -s "$output" ] && rm $output

# check all content to make sure no package is added more than once. Docs contain every docfile
if [ $TARBALL != docs ]
then
	echo "Preparing list of packages to be added the $NAME-tarball ..."
	echo "$PACKAGES" | sed "s/[[:space:]]//g;/^$/d" >> $collections_tobedone
	package_list
fi

if [ $NAME = $TARBALL ]
then

cd $texmf

# split packge
#echo "Finding fonts which are present as metafont-source(.mf), move corresponding pfb to remainder-package. Be patient ..."
##find . -type f -name '*.mf' | tee -a fontfiles
#find texmf-dist -type f -name '*.mf' > fontfiles
##sed -i -n "/amsfonts/!p" fontfiles
#rev fontfiles | cut -d'.' -f2 | cut -d'/' -f1 | rev > fontnames
#find texmf-dist -type f -name "*.pfb" > fonts.type1
#find texmf-dist -type f -name "*.pfm" >> fonts.type1
#find texmf-dist -type f -name "*.afm" >> fonts.type1
#[ -f fonts.pfb ] && rm fonts.pfb
#while read a
#do
#	grep -w "$a.pfb" fonts.type1 >> fonts.pfb
#	grep -w "$a.pfm" fonts.type1 >> fonts.pfb
#	grep -w "$a.afm" fonts.type1 >> fonts.pfb
#       #find . -type f -name "$a.pfb" >> fonts.pfb
#       #find . -type f -name "$a.pfm" >> fonts.pfb
#       #find . -type f -name "$a.afm" >> fonts.pfb
#done < fontnames
#sort -u < fonts.pfb > $tmpfile
#mv $tmpfile fonts.pfb
##sed -i "/.*amsfonts.*/d" fonts.pfb
## Only move cbfonts for now ...
#sed -i -n "/cbfonts/p" fonts.pfb
#rev fonts.pfb | cut -d'/' -f2- | rev > fontpathes
##sort -u < fontpathes > $tmpfile
##mv $tmpfile fontpathes
#while read a; do mkdir -p remainder/$a; done < fontpathes
#while read a; do mv $a remainder/$a; done < fonts.pfb
#rm fontfiles fontnames fontpathes fonts.type1 fonts.pfb

# cleanup tar-directory, just in case
[ -d texmf-dist ] && rm -rf texmf-dist
#unset flavour ; export flavour
mkdir texmf-dist &> /dev/null

VERSION=$(cat $TMP/VERSION)
case $TARBALL in
	docs)
		export flavour=".doc"
		untar $output_doc
		remove_cruft
		tar vrf $TMP/texlive-$TARBALL-$VERSION.tar texmf-dist || exit 1
		echo "Packages-list: $output_doc"
		rm -rf texmf-dist
	;;
	base|extra|fonts)
		untar $output
		remove_cruft
		tar vrf $TMP/texlive-$TARBALL-$VERSION.tar texmf-dist || exit 1
		cat $output.meta >> $output.meta.$TARBALL
		rm $output.meta
		rm $output
		rm -rf texmf-dist
	;;
esac

fi

}

# Main

case "$1" in
	base|docs|extra|fonts) TARBALL=$1; echo "Building $TARBALL tarball ..." ;;
	*) usage ;;
esac

mkdir -p $texmf
cd $TMP

# create run.tlpkg and doc.tlpkg only if $db.orig isn't there yet/was deleted
if [ ! -s $TMP/${db}.orig -o ! -s $TMP/${db} ]
then
	echo $MAJORVERSION.$(date +%y%m%d) > VERSION
	wget -c -O $TMP/${db}.orig -c ${mirror}tlpkg/$db 
	# shrink db to be faster on later processing
	sed "/^ \+./d;/^longdesc \+./d;/^cat\+./d;/^rev\+./d;/^exe\+./d;/^bin\+./d;/^src\+./d" $TMP/${db}.orig > $TMP/$db

	# as $db(might be) is new, remove the meta-files, be created again with pontentionally new content
	rm -rf $texmf/*.meta
	rm $TMP/run.tlpkg
	[ -f "$output_doc" ] && rm "$output_doc"

	# Make a list of all packages available, but exclude binary and installer/configuration packages.
	# It turns out that packagenames without '.' are what we want. Packages with '.' are all binarie-packages, which are build from source.
	grep ^name $TMP/$db | grep -v ^"name collection-" | grep -v ^"name scheme-" | grep -v '\.' | cut -d' ' -f2 > $TMP/allpackages

	# Make a list of all collections
	grep ^"name collection-" $TMP/$db | cut -d' ' -f2 > $TMP/allcollections

	# add biber (perl)binaries as special exception.
	cat <<- EOF >> $TMP/allpackages
	biber.x86_64-linux
	biber.i386-linux
	EOF

fi

# globaly excluded packages, which does not make sense without tlpkg-installer, or are non-linux specific, or are already covered by the sourcebuild,
# or are covered by an external package(asymptote), or obsolete packages(datetime replaced by datetime2, anysize replaced by geometry)

global_exclude="
"
# currently unused variable, to be considered if these are already included by the source-tarball, or strip these of the source-tarball and add them as texlive-package?
zglobal_exclude="
bibtex8
bibtexu
chktex
cjkutils
detex
dtl
dvi2tty
dvidvi
dviljk
dvipdfmx
dvipng
dvipos
dvisvgm
gsftopk
pdftools
synctex
texconfig
texlive-docindex
texlive-msg-translations
texlive-scripts
texworks

l3kernel
l3packages
l3experimental
fontspec
ocgx
luatex
"

for i in $global_exclude
do
	if [ -z "$(grep -w ^"$i"$ $TMP/allpackages)" ]
	then
		echo "\"$i\" seems not to be a tex-package listet in $db, edit the"
		echo "global_exclude variable in this script, bye."
		exit 1
	else
		sed -i "/^${i}$/d" $TMP/allpackages
		[ -s $TMP/run.tlpkg ] && sed -i "/^${i}$/d" $TMP/run.tlpkg
		[ -s $output_doc ] && sed -i "/^${i}$/d" $output_doc
	fi
done

# get linenumbers of empty lines
[ -z "$emptylines" ] && emptylines="$(grep -n ^$ $TMP/$db | cut -d':' -f1)"
# sort doc- and run- packages out to avoid binfiles and sourcfile in the texmf-tree
while read collection
do
	if [ ! -s $texmf/$collection.meta ]
	then
		package_meta
		grep ^runfiles $tmpfile &>/dev/null
	       	if [ $? = 0 ]
		then
			echo $collection >> $TMP/run.tlpkg
		fi
		grep ^docfiles $tmpfile &>/dev/null && echo $collection >> $output_doc
		mv $tmpfile $texmf/$collection.meta
	fi
done < $TMP/allpackages

# handle biber binaries to be add-able
cat << EOF >> $TMP/run.tlpkg
biber.x86_64-linux
biber.i386-linux
EOF

[ -f "$collections_done" ] && rm "$collections_done"
[ -f "$collections_tobedone" ] && rm "$collections_tobedone"

packages

# As the demanded packages are in the tarball, compress it.
echo "Compressing $TMP/texlive-$TARBALL-$VERSION.tar ..."
if [ -s $TMP/texlive-$TARBALL-$VERSION.tar ]
then
	[ -f $TMP/texlive-$TARBALL-$VERSION.tar.xz ] && rm $TMP/texlive-$TARBALL-$VERSION.tar.xz
	xz -9 -T0 $TMP/texlive-$TARBALL-$VERSION.tar || exit 1
	ls -lah $TMP/texlive-$TARBALL-$VERSION.tar.xz
fi

# cleanup
rm $tmpfile