From d880dd2b82a72946ef2bfc89a5c8164687878c11 Mon Sep 17 00:00:00 2001
From: Patrick J Volkerding <volkerdi@slackware.com>
Date: Fri, 6 Jul 2018 03:15:37 +0000
Subject: Fri Jul  6 03:15:37 UTC 2018

t/texlive-2018.180630-x86_64-1.txz:  Upgraded.
  Thanks to Johannes Schoepfer.
x/libXScrnSaver-1.2.3-x86_64-1.txz:  Upgraded.
x/libXinerama-1.1.4-x86_64-1.txz:  Upgraded.
x/libXxf86misc-1.0.4-x86_64-1.txz:  Upgraded.
---
 source/t/texlive/prep/texmf_get.sh | 856 +++++++++++++++++++++++++------------
 1 file changed, 579 insertions(+), 277 deletions(-)
 mode change 100644 => 100755 source/t/texlive/prep/texmf_get.sh

(limited to 'source/t/texlive/prep/texmf_get.sh')

diff --git a/source/t/texlive/prep/texmf_get.sh b/source/t/texlive/prep/texmf_get.sh
old mode 100644
new mode 100755
index 396fdceb..fab74ce5
--- a/source/t/texlive/prep/texmf_get.sh
+++ b/source/t/texlive/prep/texmf_get.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# texmf_get.sh (c) 2016-2017 Johannes Schoepfer, Germany, slackbuilds[at]schoepfer[dot]info
+# texmf_get.sh (c) 2016-2018 Johannes Schoepfer, Germany, slackbuilds[at]schoepfer[dot]info
 # All rights reserved.
 #
 # Redistribution and use of this script, with or without modification, is
@@ -20,16 +20,15 @@
 #  OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 #  ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
-#  V 0.14.2
+#  V 15.0.0
 #
 #  Prepare xz-compressed tarballs of texlive-texmf-trees based on texlive.tlpdb
-#  This script takes care of dependencies(as far as these are present in texlive.tlpdb) of collections and packages,
-#  and that every texlive-package is included only once.
-#  The editions(base/extra/fonts/docs) should contain nobinaries(exception biber)
-# -base: the most usefull stuff, all metafonts, all manpages, the most
-#  binaries/scripts,  65mb 2017-11-07
+#  This script takes care of dependencies(as far as these are present in texlive.tlpdb)
+#  of collections and packages, and that every texlive-package is included only once.
+#  The editions(base/extra/docs) should contain no binaries(exception biber)
+# -base: the most usefull stuff, most binaries/scripts,
+#  manpages for compiled binaries  65mb 2017-11-07
 # -docs: -base documentation only, no manpages/GNU infofiles
-# -fonts: fonts only, no metafonts 
 # -extra: remaining stuff
 #
 #  texlive netarchive policy: Every package is included as dependency 
@@ -39,231 +38,244 @@
 # package source: http://mirror.ctan.org/systems/texlive/tlnet/archive/
 
 #set -e
-MAJORVERSION=2017
+MAJORVERSION=2018
 
-# globally excluded packages, which e.g. are
+# -excluded packages, which are
 # -useless without tlmgr-installer
 # -non-linux
-# -covered by an external package
-# -obsolete 
+# -covered by an external package e.g. asymptote on SBo
+# -obsolete
+# -binaries provided already by texlive.Slackbuild
+# -does only contain sources, or hyphen directives, e.g. metatype1, patch, ...
+# -does only contain hyphen directives, e.g. hyphen-farsi ...
 
 global_exclude="
+  hyphen-farsi
+  hyphen-arabic
   asymptote
   tlcockpit
   tlshell
+  texlive.infra
+  texlive-docindex
+  texlive-scripts
+  texlive-msg-translations
   texosquery
+  texliveonfly
+  texworks
   aleph
   omega
   antomega
   omegaware
   lambda
   otibet
+  ocherokee
+  oinuit
   cslatex
+  bibarts
+  metatype1
+  patch
+  wintools.win32
+  dviout.win32
+  "
+
+  # special packages, move their type1 fonts(if metfonts are present)
+  # and/or docs to -extra
+special_packages="
+  koma-script
+  velthuis
+  armtex
+  montex
+  vntex
+  pl
+  cc-pl
+  cs
+  musixtex-fonts
+  tipa
+  cbfonts
+  ptex-fonts
+  uptex-fonts
+  "
+#Todo: split type1 fonts, or keep subset of type1 fonts in base
+#  cm-super
+
+# keep precompiled binaries, list binary, not package name 
+keep_precompiled="
+  biber
   "
 
 texmf_editions () {
-  # At first, $corepackages for the base, and other essentials
+  
+  # "excludes from -base", also dependencies are excluded
+  PACKAGES="
+    cm-super
+    biber
+    bib2gls
+    knitting
+    pgfornament
+    pgfplots
+    pst-cox
+    pst-poker
+    pst-vectorian
+    pst-geo
+    bclogo
+    $(grep ^"name .*biblatex" $db | cut -d' ' -f2 )
+    " texmfget extra || exit 1
+ 
+    # packages/collections and their dependencies for -base
   PACKAGES="
     $(cat $corepackages)
+    etoolbox
+    xcolor
+    memoir
+    velthuis
+    wasy
+    ptex
+    platex
+    revtex
+    uptex
+    uplatex
     collection-basic
     collection-latex
     collection-metapost
     collection-plaingeneric
-    collection-fontutils
     collection-luatex
     collection-context
-    cbfonts-fd
-    xetex-devanagari
-    " texmfget base || exit 1
+    collection-fontutils
+    collection-langczechslovak
+    collection-langeuropean
+    collection-langenglish
+    collection-langfrench
+    collection-langitalian
+    collection-langpolish
+    collection-langportuguese
+    collection-langspanish
+    collection-latexrecommended
+    $(collection_by_size fontsextra 70000 || exit 1)
+    $(collection_by_size publishers 10000 || exit 1)
+    " texmfget base
 
-  # Now the fonts package to make sure that big fonts like cm-super are not included elsewhere as dependency
-  PACKAGES="
-    cm-super
-    cbfonts
-    ipaex
-    wadalab
-    ethiop-t1
-    fonts-tlwg
-    uhc
-    fandol
-    arphic
-    arphic-ttf
-    nanumtype1
-    baekmuk
-    unfonts-extra
-    unfonts-core
-    " texmfget fonts || exit 1
-
-  # put some stuff in "extra" to before these make their way into "base" as dependency
+    # packages/collections and their dependencies for -extra
   PACKAGES="
-    $(grep ^"name biblatex" $db | grep -v '\.' | cut -d' ' -f2 )
-    biber.x86_64-linux
-    biber.i386-linux
-    bib2gls
-    bibarts
-    arara
-    latex2nemeth
-    ghsystem
-    adobemapping
-    knitting
-    pgfornament
-    pgfplots
+    amiri
     arabi
-    nwejm
-    uantwerpendocs
-    sduthesis
-    stellenbosch
+    arabi-add
+    arara
+    arev
+    bangorcsthesis
+    beamer2thesis
+    beamertheme-detlevcm
+    beamertheme-epyt
+    beamertheme-saintpetersburg
+    beebe
+    bhcexam
+    bxtexlogo
+    churchslavonic
+    collection-fontsextra
+    collection-langchinese
+    collection-langcjk
+    collection-langjapanese
+    collection-langkorean
+    collection-publishers
+    collection-texworks
+    collection-wintools
+    ctan-o-mat
+    ctanify
+    ctanupload
+    dad
+    ethiop-t1
+    fibeamer
     fithesis
+    ghsystem
     gregoriotex
+    hustthesis
+    ijsra
+    iwhdp
+    jpsj
+    kerkis
+    ketcindy
+    kpfonts
+    langsci
+    latex2nemeth
+    libertine
     lilyglyphs
-    musixtex-fonts
-    beebe
-    velthuis
+    lni
+    luatexko
+    media9
+    musuos
     mwe
+    newtx
+    nwejm
+    padauk
+    pdfwin
     pdfx
-    media9
-    pst-cox
-    pst-poker
-    pst-vectorian
-    pst-geo
+    powerdot-tuliplab
+    powerdot-FUBerlin
     quran
-    ijsra
-    fibeamer
-    udesoftec
-    xduthesis
-    hustthesis
-    bangorcsthesis
-    sapthesis
-    uowthesis
-    cs
-    pl
-    cc-pl
-    tipa
-    kerkis
-    amiri
-    cns
-    vntex
-    montex
-    xcharter
-    fonts-churchslavonic
-    japanese-otf
+    resumecls
     sanskrit-t1
+    sapthesis
+    sduthesis
+    seuthesix
+    simurgh
     skaknew
-    padauk
+    stellenbosch
+    suanpan
+    tudscr
+    uantwerpendocs
+    ucs
+    udesoftec
+    universalis
+    uowthesis
+    wasy2-ps
+    xduthesis
+    xetexko 
+    xq
     " texmfget extra || exit 1
-
-  # Completing the base
+ 
+   # What's left, for base again 
   PACKAGES="
+    collection-fontsrecommended
+    collection-xetex
     collection-langcyrillic
-    collection-langczechslovak
-    collection-langeuropean
-    collection-langenglish
-    collection-langfrench
+    collection-langarabic
     collection-langgerman
     collection-langgreek
-    collection-langitalian
-    collection-langpolish
-    collection-langportuguese
-    collection-langspanish
-    collection-langjapanese
-    collection-langkorean
-    collection-langarabic
-    collection-langchinese
-    collection-langcjk
     collection-langother
-    collection-xetex
     collection-humanities
     collection-mathscience
     collection-pictures
-    collection-publishers
+    collection-pstricks
     collection-music
     collection-games
-    collection-fontsrecommended
-    collection-latexrecommended
     collection-binextra
     collection-bibtexextra
     collection-formatsextra
     collection-latexextra
-    acro
-    acronym
-    acroterm
-    enumitem
-    enumitem-zref
-    yfonts
-    doublestroke
-    was
-    xypic
-    barcodes
-    qrcode
-    lastpage
-    appendix
-    changebar
-    footmisc
-    multirow
-    overpic
-    subfigure
-    titlesec
-    siunitx
-    combelow
-    csquotes
-    etoolbox
-    etextools
-    idxlayout
-    bidi
-    filecontents
-    eplain
-    texsis
-    mltex
-    lollipop
-    moreverb
-    indextools
-    splitindex
-    eepic
-    bigfoot
-    xstring
-    showexpl
-    cweb-latex
-    hypdvips
-    ptex
-    perltex
-    collection-pstricks
-    $(collection_by_size fontsextra 20000 || exit 1)
     " texmfget base || exit 1
 
-  # Put all remaining stuff in "extra" 
-  # Pull some bin-packages from tlnet, which aren't provided by the texlive source tarball
-  PACKAGES="
-    collection-texworks
-    collection-wintools
-    wasy2-ps
-    " texmfget extra || exit 1
-
-  # Call "fonts" at the end to add remaining fonts
-  PACKAGES="
-    collection-fontsextra
-    " texmfget fonts || exit 1
-
 }
 
 # ==== Nothing to edit beyond this line ====
 
 usage () {
   echo
-  echo "Generate texmf trees/editions based on collections/packages and their dependencies."
-  echo "./texmf_get.sh [base|docs|extra|fonts]"
+  echo "Generate texmf trees/editions based on collections/packages"
+  echo "and their (hard)dependencies."
+  echo "./texmf_get.sh [base|docs|extra|lint]"
   echo
   echo "-base:  texfiles, no docs"
   echo "-docs:  docs of -base"
   echo "-extra: remaining texfiles and docs"
-  echo "-fonts: fonts and docs" 
+  echo "[lint]: compare filename contents of all generated editions,"
+  echo " to detect overlapping files"
   echo 
   echo "Only new/updated/missing tex packages are downloaded."
   echo "The first run takes \"long\", tex packages(about 2500Mb)"
-  echo "need to be downloaded and metafiles are generated."
+  echo "need to be downloaded."
   echo "To check out a new version/release, delete"
   echo "$db"
-  echo "A new ascii database is pulled on the next run,"
-  echo "and a new version yymmdd is set."
+  echo "A new ascii index file/database(texlive.tlpdb) is will be"
+  echo "pulled on the next run, and a new version yymmdd will be set."
   echo
   echo "All generated tarballs, logs etc. are going to"
   echo "$TMP"
@@ -362,36 +374,116 @@ download () {
 }
 
 untar () {
-  # leave if $1 has no content. Therea collections with basically no used package, e.g. texworks
+  # leave if $1 has no content
   if [ -s "$1" ]
   then
     while read package
     do
+      echo "untar $package"
       # untar all packages, check for relocation, "relocate 1" -> untar in texmf-dist
       download $package || exit 1
-      # untar the tex package
-      unset relocated
-      [ -n "$(grep -w ^"relocated 1" $texmf/$package.meta)" ] && relocated="-C texmf-dist" 
-      tar vxf ${package}${flavour}.tar.xz --exclude tlpkg $relocated || exit 1
+      # untar the tex package, relocate to texmf-dist if necessary, binpackages always need relocation
+      relocated='.'
+      [ -n "$(grep -w ^"relocated 1" $texmf/$package.meta)" -o -n "$(grep ^"binfiles " $texmf/$package.meta)" ] && relocated="texmf-dist" 
+      # if not .doc package, investigate files for dependencies/provides
+      if [ -n "$flavour" ]
+      then
+        tar xf ${package}${flavour}.tar.xz --exclude tlpkg -C $relocated || exit 1
+      else
+        #tar vxf ${package}${flavour}.tar.xz --exclude tlpkg -C $relocated 
+        tar vxf ${package}${flavour}.tar.xz --exclude tlpkg -C $relocated | egrep '\.sty$|\.bbx$|\.cls$' > $texmf/$package.deps
+        if [ -n "$texmf/$package.deps" ]
+        then
+          unset provide
+          unset depends
+          for depfile in $(cat $texmf/$package.deps)
+          do
+            filename="$( echo $depfile | rev | cut -d'.' -f2- | cut -d'/' -f1 | rev)"
+            # always add $filename as "ProvidesPackage", if it's a .sty
+            echo $depfile | grep '\.sty'$ &>/dev/null
+            [ $? = 0 ] && provide+="${filename},"
+            # remove comments, if there are backslashes ignore that content, except it is \filename
+            #provide+="$(sed "s/%.*//g" $texmf/$relocated/$depfile | sed -n "s/.*\\\ProvidesPackage{\([^}]*\)}.*/\1/p" | sed "s/\\\filename/$filename/g;/\\\/d" | sort -u | tr '\n' ',')"
+            #sed -z "s/.*\\\Provides\(Package\|ExplPackage\|File\|Class\)*.\n//g" | sed "s/[[:space:]]//
+           #provide+="$(sed "s/%.*//g" $texmf/$relocated/$depfile | sed -z "s/.*\\\Provides\(Package\|ExplPackage\|File\|Class\)*.\n//g" | sed "s/[[:space:]]// | sed -n "s/.*\\\Provides\(Package\|ExplPackage\|File\|Class\){\([^}]*\)}.*/\2/p" | sed "s/\\\filename/$filename/g;s/\\\ExplFileName/$filename/g;/\\\/d" | sed "s/\(\.sty$\|\.cls$\)//g" | sort -u | tr '\n' ',')"
+            provide+="$(sed "s/%.*//g" $texmf/$relocated/$depfile | sed -z "s/\(Package\|ExplPackage\|File\|Class\)\n/\1/g" | sed "s/[[:space:]]//" | sed -n "s/.*\\\Provides\(Package\|ExplPackage\|File\|Class\){\([^}]*\)}.*/\2/p" | sed "s/\\\filename/$filename/g;s/\\\ExplFileName/$filename/g" | sed "s/\(\.sty$\|\.cls$\)//g" | sort -u | tr '\n' ',')"
+            #depends+="$(sed "s/%.*//g" $texmf/$relocated/$depfile | sed -n "s/.*\(\\\require\|\\\use\)package{\([^}]*\)}.*/\2/p" | sed "/\\\/d" | sort -u | tr '\n' ',')"
+            depends+="$(sed "s/%.*//g" $texmf/$relocated/$depfile | sed -n "s/.*\(\\\require\|\\\use\)package{\([^}]*\)}.*/\2/p" | sort -u | tr '\n' ',')"
+            #depends+="$(sed "s/%.*//g" $texmf/$relocated/$depfile | sed -n "s/.*\\\\(require\|use\)package{\([^}]*\)}.*/\2/p" | sed "/\\\/d" | sort -u | tr '\n' ',')"
+          done
+          if  [ -n "$provide" ]
+          then
+            #echo "$package $provide | sort -u | tr '\n' ',' " >> $TMP/provides.run.$edition
+            echo "$package $provide" >> $TMP/provides.run.$edition
+          fi
+          if [ -n "$depends" ]
+          then
+            echo "$package $depends" >> $TMP/depends.run.$edition
+          fi
+        fi
+      fi
+
+      # Check for binaries, delete them as these should be provided
+      #  by texlive.Slackbuild, keep symlinks and scripts
       
-      # if binaries are present, put them in texmf-dist
-      [ -d bin ] && cp -a bin texmf-dist && rm -rf bin
+      unset binaries
+      unset scripts
+      for arch in $platforms
+      do
+        if [ -d $texmf/texmf-dist/bin/$arch ]
+        then
+          [ ! -d $texmf/texmf-dist/linked_scripts ] \
+            && mkdir $texmf/texmf-dist/linked_scripts 
+          # rewrite link target to fit systemwide installation
+          for link in $(find $texmf/texmf-dist/bin/$arch -type l)
+          do
+            ln -sf $(readlink $link | sed "s/^..\/..\(.*\)/..\/share\1/" ) $link || exit 1
+          done
+          # move symlinks/scripts to linked_scripts
+          find $texmf/texmf-dist/bin/$arch -type l -exec mv '{}' $texmf/texmf-dist/linked_scripts/ \; 
+        
+          binaries="$(find $texmf/texmf-dist/bin/$arch -type f -exec file '{}' + | grep -e "executable" -e "shared object" | grep ELF | cut -f 1 -d : )"
+          for bin in $binaries
+          do
+            # keep binaries of special packages
+            for binary in $keep_precompiled
+            do
+              if [ "$(echo $bin | rev | cut -d'/' -f1 | rev )" != "$binary" ]
+              then
+                rm $bin
+                echo $bin | rev | cut -d'/' -f1 | rev >> $binary_removed.$edition
+              fi
+            done
+          done
+          # move scripts to linked-scripts
+          scripts="$(find $texmf/texmf-dist/bin/$arch -type f -exec file '{}' + | grep -wv ELF | cut -f 1 -d : )"
+          for script in $scripts
+          do
+            mv $script $texmf/texmf-dist/linked_scripts/ 
+          done
+        fi
+      done
+
       if [ "$flavour" = ".doc" ]
       then
-        size=$(( $(grep ^doccontainersize $texmf/$package.meta | cut -d' ' -f2 ) / 1024 ))
+        size=$(grep ^doccontainersize $texmf/$package.meta | cut -d' ' -f2) 
       else
-        size=$(( $(grep ^containersize $texmf/$package.meta | cut -d' ' -f2 ) / 1024 ))
+        size=$(grep ^containersize $texmf/$package.meta | cut -d' ' -f2)
       fi
       shortdesc="$(grep ^shortdesc $texmf/$package.meta | cut -d' ' -f2- )"
-      echo "$size Kb, $package$flavour: $shortdesc" >> $output.meta
-      #grep ^"execute addMap" $texmf/$package.meta | sed "s/^execute //g" >> $output.updmap.cfg
+      echo "$size byte, $package$flavour: $shortdesc" >> $output.meta
+      # make index of uncompressed size of each package
+      echo "$(xz -l --verbose ${package}${flavour}.tar.xz | grep "Uncompressed size" | cut -d'(' -f2 | cut -d' ' -f1 ) byte, $package$flavour: $shortdesc" >> $output.meta.uncompressed
     done < $1
     
     # copy packages index to texmf-dist, so included packages are known in later installation
-    cat $output.meta >> $output.$edition.meta
+    # don't list binary packages, as the binaries itself are not contained, only the symlinks.
+    cat $output.meta | grep -v '\-linux:'  >> $output.$edition.meta
+    cat $output.meta.uncompressed | grep -v '\-linux:' >> $output.$edition.meta.uncompressed
     
     # cleanup
     [ -f $output.meta ] && rm $output.meta
+    [ -f $output.meta.uncompressed ] && rm $output.meta.uncompressed
   fi
 }
 
@@ -399,19 +491,24 @@ remove_cruft () {
   # Remove m$-stuff, ConTeXt single-user-system stuff, source leftovers and pdf-versions of manpages
   rm -rf texmf-dist/source
   rm -rf texmf-dist/scripts/context/stubs/source/
-  find texmf-dist/ -type d -name 'win32'     -exec rm -rf {} +
-  find texmf-dist/ -type d -name 'win64'     -exec rm -rf {} +
-  find texmf-dist/ -type d -name 'mswin'     -exec rm -rf {} +
-  find texmf-dist/ -type d -name 'win'       -exec rm -rf {} +
-  find texmf-dist/ -type d -name 'setup'     -exec rm -rf {} +
-  find texmf-dist/ -type d -name 'install'   -exec rm -rf {} +
-  find texmf-dist/ -type f -name '*.bat'     -delete
-  find texmf-dist/ -type f -name '*.bat.w95' -delete
-  find texmf-dist/ -type f -name '*win32*'   -delete
-  find texmf-dist/ -type f -name 'winansi*'  -delete
-  find texmf-dist/ -type f -name '*-man.pdf' -delete
+  find texmf-dist/ -type d -name 'win32'         -exec rm -rf {} +
+  find texmf-dist/ -type d -name 'win64'         -exec rm -rf {} +
+  find texmf-dist/ -type d -name 'mswin'         -exec rm -rf {} +
+  find texmf-dist/ -type d -name 'win'           -exec rm -rf {} +
+  find texmf-dist/ -type d -name 'setup'         -exec rm -rf {} +
+  find texmf-dist/ -type d -name 'install'       -exec rm -rf {} +
+  find texmf-dist/ -type f -name 'uninstall*.sh' -delete
+  find texmf-dist/ -type f -name '*.bat'         -delete
+  find texmf-dist/ -type f -name '*.bat.w95'     -delete
+  find texmf-dist/ -type f -name '*win32*'       -delete
+  find texmf-dist/ -type f -name 'winansi*'      -delete
+  find texmf-dist/ -type f -name '*man1.pdf'     -delete
+  find texmf-dist/ -type f -name '*man5.pdf'     -delete
   # Remove zero-length files, as these appear e.g. in hyph-utf8 tex-package.
-  find . -type f -size 0c -delete
+  # find texmf-dist/ -type f -size 0c -delete
+  find texmf-dist/ -type f -empty -delete
+  # Remove empty directories recursively
+    find texmf-dist/ -type d -empty -delete
 }
 
 texmfget () {
@@ -444,31 +541,64 @@ texmfget () {
     fi
     
     package_meta $collection || exit 1
-  
-    # Don't handle collections as dependency of other collections, as this destroys control over what packages to be added
+ 
     # If $collection is a singel package(not a collection-), add it here
     if [ -n "$(head -n1 $texmf/$collection.meta | fgrep -v "name collection" )" ]
     then
-      # if package contains only docs, add to docpackages
-      if [ -z "$(grep ^runfiles $texmf/$collection.meta)" -a -n "$(grep ^docfiles $texmf/$collection.meta)" ]
+      addpackage=no 
+      # if package contains docs, add to docpackages
+      if [ -n "$(grep ^docfiles $texmf/$collection.meta)" ]
       then
-        sed -i "/^$collection$/d" $collections_tobedone
-        echo "$collection" >> $collections_done
         echo "$collection" >> $output_doc
-        echo "$collection added to -docs $1" >> $logfile
-        continue
+        echo "$collection added to docs $1" >> $logfile
+        addpackage=yes 
       fi
-      # if package contains also docs, add also to docpackages
-      if [ -n "$(grep ^docfiles $texmf/$collection.meta)" ]
+      if [ -n "$(grep ^runfiles $texmf/$collection.meta)" -o -n "$(grep ^binfiles $texmf/$collection.meta)" ]
       then
-        echo "$collection" >> $output_doc
-        echo "$collection added to -docs $1" >> $logfile
+        echo "$collection" >> $output
+        echo "$collection added to -$1" >> $logfile
+        addpackage=yes 
+      fi
+      # very package should be added to one edition, abort if that didn't work
+      if [ $addpackage = no ]
+      then
+        echo "$collection doesn't contain any docfiles/runfiles/binfiles"
+        echo "Please exclude package/report to upstream mailinglist tex-live@tug.org, bye."
+        exit 1
       fi
-      echo "$collection" >> $output
-      echo "$collection added to -$1" >> $logfile
     fi
+
+    # Don't handle collections as dependency of other collections, as this destroys control over what packages to be added
     # add dependend packages, but no binary(ARCH) and no packages conataining a '.'. Packges with dot indicate binary/texlive-manager/windows packages
-    grep ^"depend " $texmf/$collection.meta | grep -v "ARCH$" | grep -v '\.' | cut -d' ' -f2- > $dependencies
+    
+    grep ^"depend " $texmf/$collection.meta | cut -d' ' -f2- > $dependencies
+    
+    if [ -s "$dependencies" ]
+    then
+      # check for .ARCH packages which may be binaries, scripts or links
+      # Binaries should all come from the sourcebuild(exception biber)
+      for dependency in $(cat $dependencies)
+      do
+        echo $dependency | grep '\.ARCH'$ &>/dev/null
+        if [ $? = 0 ] 
+        then
+          for arch in $platforms
+          do
+            archpackage="$(echo $dependency | sed "s/\.ARCH$/\.$arch/")"
+            grep ^"name $archpackage"$ $db &>/dev/null && echo "$archpackage" >> $dependencies.verified_arch
+          done
+        else
+          echo $dependency >> $dependencies.verified_arch
+        fi
+      done
+      if [ -f $dependencies.verified_arch ]
+      then
+        mv $dependencies.verified_arch $dependencies
+      else
+        rm $dependencies
+      fi
+    fi
+
     if [ -s "$dependencies" ]
     then
       echo "----------------" >> $logfile
@@ -502,7 +632,7 @@ texmfget () {
   # handle doc package index, one for each edition
   cat $output_doc >> $TMP/packages.$1.doc
 
-  # untar only one $edition, untar docs together with -extra/-fonts edition
+  # untar only one $edition, untar docs together with -extra edition
   if [ "$1" = $edition -o docs = $edition ]
   then
     cd $texmf
@@ -514,22 +644,23 @@ texmfget () {
     # Make tarball/checksum reproducible by setting mtime(clamp-mtime), owner, group and sort content
     # --clamp-mtime --mtime doesn't work with tar 1.13, when makepkg creates the tarball:
     # tar-1.13: time_t value 9223372036854775808 too large (max=68719476735)
+    echo "Adding files to $( echo $tarball | rev | cut -d'/' -f1 | rev ) ..."
     case $edition in
       base)
       unset flavour
       untar $output || exit 1
       remove_cruft || exit 1
-      tar vrf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
+      tar rf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
       rm -rf texmf-dist
       ;;
-      extra|fonts)
+      extra)
       unset flavour
       untar $output || exit 1
       export flavour=".doc"
       untar $output_doc || exit 1
       remove_cruft || exit 1
       #tar vrf $tarball --clamp-mtime --mtime --owner=0 --group=0 --sort=name texmf-dist || exit 1
-      tar vrf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
+      tar rf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
       rm -rf texmf-dist
       ;;
       docs)
@@ -540,7 +671,7 @@ texmfget () {
         untar $output_doc || exit 1
         remove_cruft || exit 1
       #tar vrf $tarball --clamp-mtime --mtime --owner=0 --group=0 --sort=name texmf-dist || exit 1
-        tar vrf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
+        tar rf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
         rm -rf texmf-dist
       fi
       ;;
@@ -548,6 +679,48 @@ texmfget () {
   fi
 }
 
+lint () {
+
+echo "Comparing content of all editions, this may take a while ..."; 
+cd $TMP
+# check if all editions of same VERSION are there, take -base as reference
+lint_version=$( ls texlive-base-*tar.xz | head -n1 | cut -d'.' -f2 || exit 1)
+if [ -s texlive-extra-$MAJORVERSION.$lint_version.tar.xz \
+  -a -s texlive-docs-$MAJORVERSION.$lint_version.tar.xz  ]
+then
+  for edition in base extra docs
+  do
+    echo "Extracting index of texlive-${edition}-$MAJORVERSION.$lint_version.tar.xz ..."
+    # don't list directories
+    tar tf texlive-${edition}-$MAJORVERSION.$lint_version.tar.xz | grep -v '/'$ > $TMP/packages.$edition.lint
+  done
+
+  # compare content
+  for edition in base extra docs
+  do
+    >$TMP/packages.$edition.lint.dup
+    case $edition in
+      base)
+        echo "check if files of base are present in another edition"
+        while read line
+        do
+          grep ^"$line"$ $TMP/packages.extra.lint >> $TMP/packages.base.lint.dup
+          grep ^"$line"$ $TMP/packages.docs.lint >> $TMP/packages.base.lint.dup
+        done < $TMP/packages.$edition.lint
+      ;;
+    esac
+  done
+else
+  echo "Not all editions are present to lint them. Create them first by"
+  echo "$0 [base|docs|extra]"
+  echo "bye."
+  exit 1
+fi
+
+exit 0
+
+}
+
 # Main
 
 # release mirror
@@ -566,31 +739,36 @@ collections_done=$TMP/done
 collections_tobedone=$TMP/tobedone
 corepackages=$TMP/corepackages
 allcollections=$TMP/allcollections
-metafonts=$TMP/metafonts
+binary_removed=$TMP/binaries.removed
 manpages=$TMP/manpages
 dependencies=$TMP/deps
 packages_base=$TMP/packages.base
 packages_extra=$TMP/packages.extra
-packages_fonts=$TMP/packages.fonts
-packages_metafont=$TMP/packages.metafont
 packages_manpages=$TMP/packages.manpages
+files_split=$TMP/files.split
+platforms="x86_64-linux i386-linux"
+
+mkdir -p $texmf
+cd $TMP
 
 case "$1" in
-  base|docs|extra|fonts) edition=$1; echo "Building $edition tarball ...";;
+  base|docs|extra) edition=$1;;
+  lint) lint ;;
   *) usage; exit 0 ;;
 esac
-  
-mkdir -p $texmf
-cd $TMP
+
+echo "Building $edition tarball ..."
 
 # Set VERSION, get texlive.tlpdb and keep unshorten $db.orig 
-if [ ! -s ${db}.orig -o ! -s $db ]
+if [ ! -s ${db}.orig -o ! -s $db -o ! -s VERSION ]
 then
   echo $MAJORVERSION.$(date +%y%m%d) > VERSION
   wget -c -O ${db}.orig ${mirror}tlpkg/texlive.tlpdb 
   # remove most content from $db to be faster on later processing. 
-  # keep dependencies/manpages/metafonts/binfiles/shortdesc/sizes
-  egrep '^\S|^ RELOC/doc/man|^ texmf-dist/doc/man/man|^ RELOC/doc/info/|^ texmf-dist/doc/info/|^ texmf-dist/fonts/source/public/|^ RELOC/fonts/source/public|^ bin|^$' ${db}.orig | grep -v ^longdesc > $db
+  # keep dependencies/manpages/binfiles/shortdesc/sizes
+  egrep \
+    '^\S|^ RELOC/doc/man|^ texmf-dist/doc/man/man|^ RELOC/doc/info/|^ texmf-dist/doc/info/|^ bin|^$' \
+    ${db}.orig | grep -v ^longdesc > $db
   
   # As $db (might be)/is new, remove the meta-files, might created again with (pontentionally) new content
   rm -rf $texmf/*.meta
@@ -604,7 +782,19 @@ grep -B1 ^'category TLCore' $db | grep -v ^'category TLCore' |  grep -v ^-- | gr
 
 # Make a list of all collections
 grep ^"name collection-" $db | cut -d' ' -f2 > $allcollections
-  
+
+# translate .ARCH to platforms in excludes, to make .ARCH packages excludeable by $global_exclude
+for exclude in $global_exclude
+do
+  if [ -n "$(echo $exclude | grep '\.ARCH'$ )" ]
+  then
+    for arch in $platforms
+    do
+      global_exclude+=" $(echo $exclude | sed "s/\.ARCH$/\.$arch/")" 
+    done
+    global_exclude=${global_exclude/$exclude/}
+  fi
+done
 # globally exclude from $corepackages
 for exclude in $global_exclude
 do
@@ -620,36 +810,34 @@ logfile=$TMP/$VERSION.log
 >$logfile
 >$tarball
 >$collections_done
->$metafonts
+>$files_split
 >$manpages
->$packages_metafont
 >$packages_manpages
 >$packages_base
 >$packages_extra
->$packages_fonts
 >$packages_base.doc
 >$packages_extra.doc
->$packages_fonts.doc
 >$TMP/packages.$edition.meta
+>$TMP/packages.$edition.meta.uncompressed
+>$TMP/provides.run.$edition
+>$TMP/depends.run.$edition
+>$binary_removed.$edition
 
-# put the editions base/extra/fonts together
+# put the editions base/extra together
 texmf_editions || exit 1
 
-# Check if all collections ar part in at least one edition
+# Check if all collections are part in at least one edition
 while read collection
 do
   grep -w "$collection" $collections_done &> /dev/null
   if [ $? != 0 ]
   then
-    echo "Error: $collection was not handled, edit packages/collections inthe  texmfget function in $0." | tee -a $logfile
+    echo "Error: $collection was not handled."
+    echo "Edit packages/collections in the texmfget function." | tee -a $logfile
     exit 1
   fi
 done < $allcollections
 
-# meta data about added packages
-sort -n $output.$edition.meta > $tmpfile
-mv $tmpfile $output.$edition.meta 
-
 # cleanup 
 rm $allcollections
 rm $corepackages
@@ -659,89 +847,203 @@ rm $output
 rm $output_doc
 rm $dependencies
 
+# untar special- and manpage packages to be splitted/moved to other editions
+# splitting special packages, files index
+echo "Prepare index of to be splitted/moved files from -base"
 [ ! -d texmf-dist ] && mkdir texmf-dist
-# include all metafonts in base package, plus packages which misses font-mf tag on CTAN
-echo "Looking for metafont files to be included in -base ..."
-for metafont in $(paste -s $packages_extra $packages_fonts )
+for package in $special_packages
 do
-  if [ -n "$(egrep "(fonts/source/public/)" $texmf/$metafont.meta )" ]
+  echo "Splitting $package"
+  # special packages have to be in -base, as only here are special
+  # tasks done to reduce size of -base edition
+  if [ -z "$( grep ^"$package"$ $packages_base )" ]
   then
-    # include all metafonts in -base, write index for later exclution from other editions.
-    package_meta $metafont || exit 1
-    echo "Adding metafonts from $metafont to -base"
-    unset flavour
-    download $metafont || exit 1
-    unset relocated
-    pathprefix="texmf-dist/"
-    [ -n "$(grep -w ^"relocated 1" $texmf/$metafont.meta)" ] && relocated="-C texmf-dist" && unset pathprefix
-    tar vxf $texmf/$metafont.tar.xz $relocated ${pathprefix}fonts/source ${pathprefix}tex/latex 2>/dev/null | sed "s/^fonts/texmf-dist\/fonts/g;s/^tex\//texmf-dist\/tex\//g" >> $metafonts
-    echo $metafont >> $packages_metafont
+    echo "$package was not found to be part of -base"
+    echo "Edit \$special_packages in $0"
+    echo "to contain only packages from -base, bye."
+    exit 1
   fi
+  unset relocated
+  pathprefix="texmf-dist/"
+  [ -n "$(grep -w ^"relocated 1" $texmf/$package.meta)" ] && \
+    relocated="-C texmf-dist" && unset pathprefix
+  # avoid big pdf docs which are also present as html
+  # move (big)type1 fonts to -extra
+  # $files_split lists files to be moved from -base -extra
+  if [ $package = "cm-super" ]
+  then
+    # cm-super minimal for -base, create index of extended cm-super
+    tar tf $texmf/$package.tar.xz | sed \
+      "/1000\.pfb$/d;/^tlpkg/d;/\.sty$/d;/\.enc$/d;/\.GS$/d" \
+      | tee -a $files_split > $files_split.tmp
+  else
+    tar tf $texmf/${package}.tar.xz | sed \
+    -ne "/.*doc\/latex\/.*\.pdf$/p" \
+    -ne "/.*fonts\/map\/.*\.map$/p" \
+    -ne "/.*fonts\/enc\/.*\.enc$/p" \
+    -ne "/.*fonts\/afm\/.*\.\(afm\|afm\.gz\)$/p" \
+    -ne "/.*fonts\/type1\/.*\.pfb$/p" \
+    -ne "/.*fonts\/vf\/.*\.vf$/p" \
+    | tee -a $files_split > $files_split.tmp
+  fi
+
+  if [ $edition = base ]
+  then
+    # Calculate package-minimal size, uncompressed and compressed
+    mkdir -p calculate/texmf-dist
+    tar xf $texmf/$package.tar.xz -C calculate/texmf-dist --exclude-from=$files_split.tmp
+    tar cf calculate/calc.tar.xz -I 'xz -9' calculate/texmf-dist
+    size_minimal=$(du -bc calculate/calc.tar.xz | tail -n1 | sed "s/[[:space:]].*//")
+    size_minimal_uncompressed="$(xz -l --verbose calculate/calc.tar.xz | grep "Uncompressed size" | cut -d'(' -f2 | cut -d' ' -f1 )"
+    sed -i \
+      -e "s/^[0-9]* byte, $package: /$size_minimal byte, $package-minimal: /" \
+      $output.base.meta
+    sed -i \
+      -e "s/^[0-9]* byte, $package: /$size_minimal_uncompressed byte, $package-minimal: /" \
+      $output.base.meta.uncompressed
+    rm -rf calculate
+  fi
+  
+  if [ $edition = extra ]
+  then
+    mkdir -p calculate/texmf-dist
+    tar xf $texmf/${package}.tar.xz -C calculate/texmf-dist $(paste $files_split.tmp)
+    tar cf calculate/calc.tar.xz -I 'xz -9' calculate/texmf-dist
+    size_extended=$(du -bc calculate/calc.tar.xz | tail -n1 | sed "s/[[:space:]].*//")
+    size_extended_uncompressed="$(xz -l --verbose calculate/calc.tar.xz | \
+      grep "Uncompressed size" | cut -d'(' -f2 | cut -d' ' -f1 )"
+ 
+    # put new sizes in package index uncompressed
+    sed -i \
+      -e "s/^[0-9]* byte, $package: /$size_extended byte, $package-extended: /" \
+      $output.extra.meta
+    sed -i \
+      -e "s/^[0-9]* byte, $package: /$size_extended_uncompressed byte, $package-extended: /" \
+      $output.extra.meta.uncompressed
+    rm -rf calculate
+  fi
+ 
+  # untar to provide files for -extra
+  tar xf $texmf/${package}.tar.xz $relocated $(paste $files_split.tmp)
+  if [ $package = "cm-super" ]
+  then
+    # create cm-super- minimal config/maps with 10pt glyphs only
+    sed "s/cm-super/cm-super-minimal/g" $texmf/texmf-dist/dvips/cm-super/config.cm-super \
+      > $texmf/texmf-dist/dvips/cm-super/config-minimal.cm-super
+    for map in t1 t2a t2b t2c ts1 x2
+    do
+      grep 1000 $texmf/texmf-dist/fonts/map/dvips/cm-super/cm-super-$map.map \
+        > $texmf/texmf-dist/fonts/map/dvips/cm-super/cm-super-minimal-$map.map 
+      sed -i "/.*1000\.pfb/d" $texmf/texmf-dist/fonts/map/dvips/cm-super/cm-super-$map.map 
+    done
+  fi
+
 done
 
+
+# cleanup
+rm $files_split.tmp
+# fix relocation in index for splitted packages
+sed -i \
+  -e "s|^doc|texmf-dist\/doc|g" \
+  -e "s|^fonts|texmf-dist\/fonts|g" \
+  -e "s|^dvips|texmf-dist\/dvips|g" \
+  $files_split
+
+# sort meta data about added packages
+sort -n $output.$edition.meta > $tmpfile
+mv $tmpfile $output.$edition.meta 
+sort -n $output.$edition.meta.uncompressed > $tmpfile
+mv $tmpfile $output.$edition.meta.uncompressed 
+
+sort -u $binary_removed.$edition > $tmpfile
+mv $tmpfile $binary_removed.$edition
+
 # include manpages/GNU infofiles in -base, write index for later exclution from other editions.
-# In -extra there should not be any manpage left.
+# In -extra/-docs there should not be any manpage left.
 echo "Looking for manpages/GNU infofiles to be included in -base ..."
-for package in $(paste -s $packages_metafont $packages_base.doc | sort -u)
+for package in $(paste -s $packages_base.doc | sort -u)
 do
   if [ -n "$(egrep "(doc/man/man|doc/info/)" $texmf/$package.meta )" ]
   then
-    echo "Adding manpage from $package to -base"
+    echo "Adding manpage from $package.doc to -base"
     flavour=".doc" download $package || exit 1
     unset relocated
     pathprefix="texmf-dist/"
-    [ -n "$(grep -w ^"relocated 1" $texmf/$package.meta)" ] && relocated="-C texmf-dist" && unset pathprefix
-    tar vxf $texmf/${package}.doc.tar.xz --exclude "*.man[15].pdf" $relocated ${pathprefix}doc/man/ ${pathprefix}doc/info 2>/dev/null | sed "s/^doc/texmf-dist\/doc/g" >> $manpages
+    [ -n "$(grep -w ^"relocated 1" $texmf/$package.meta)" ] \
+      && relocated="-C texmf-dist" && unset pathprefix
+    tar tf $texmf/${package}.doc.tar.xz | sed \
+    -ne "/.*doc\/man\/.*\.1$/p" \
+    -ne "/.*doc\/man\/.*\.5$/p" \
+    -ne "/.*doc\/info\/.*\.info$/p" \
+    | tee -a $manpages > $manpages.tmp
+    # untar to provide files for -/extra/-docs
+    tar xf $texmf/${package}.doc.tar.xz $relocated $(paste $manpages.tmp)
     echo "$package" >> $packages_manpages
   fi
 done 
+# cleanup
+rm $manpages.tmp
+sed -i \
+  -e "s/^doc/texmf-dist\/doc/g" \
+  $manpages
   
 case $edition in
   base) 
   # Content info
   cat << EOF | gzip -9 >> $texmf/texmf-dist/packages.$edition.gz
 Content of -$edition:
-$(sort $packages_base)
-
-Metafonts from packages:
-$(sort $packages_metafont)
-
-Manpages from packages:
-$(sort $packages_manpages)
+$(sed "/-linux$/d" $packages_base | sort)
 EOF
-  
-  # add manpages/metafonts to the tarball
-  tar rf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
-  # cleanup extracted metafonts/manpages
-  rm -rf texmf-dist
-  # handle koma-script docs, the author wants the docs to be shipped along, html doc seems sufficient
-  tar f $tarball --delete $(tar tf $tarball | grep /doc/.*koma-script.*pdf)
+#Splitted packages, type1 fonts/docs moved to -extra:
+#$(echo $special_packages)
+#EOF
+
+# add manpages/GNU infofiles to the tarball
+  tar rf $tarball --owner=0 --group=0 --sort=name \
+    texmf-dist/doc/man/ texmf-dist/doc/info/ \
+    texmf-dist/packages.$edition.gz \
+    || exit 1
+#  # add cm-super minimal maps/config
+#  tar rf $tarball --owner=0 --group=0 --sort=name \
+#    texmf-dist/dvips/cm-super/config-minimal.cm-super \
+#    --wildcards texmf-dist/fonts/map/dvips/cm-super/cm-super-minimal-*.map \
+#    || exit 1
+  echo "Removing files -from base, splitted from special packages to be included in -extra"
+  tar f $tarball --delete $(paste $files_split) || exit 1
   ;;
-  extra|fonts)
-  # cleanup extracted metafonts/manpages
-  rm -rf texmf-dist
-  echo "Removing manpages/metafonts from -extra/-fonts/-docs which now reside in -base" 
-  tar -f $tarball --delete $(paste $manpages $metafonts) 2>/dev/null
+  extra)
+  echo "Removing manpages from $edition which now reside in -base" 
+  tar f $tarball --delete $(paste $manpages) 2>/dev/null 
   # content info
-  mkdir texmf-dist
   echo "Content of -$edition, including documentation:" > $texmf/texmf-dist/packages.$edition
-  sort $TMP/packages.$edition >> $texmf/texmf-dist/packages.$edition
+  sed "/-linux$/d" $TMP/packages.$edition | sort >> $texmf/texmf-dist/packages.$edition
   gzip -9 $texmf/texmf-dist/packages.$edition
-  tar rf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
+#  # remove cm-super minimal config, which resides in -base
+#  rm \
+#    $texmf/texmf-dist/dvips/cm-super/config-minimal.cm-super \
+#    $texmf/texmf-dist/fonts/map/dvips/cm-super/cm-super-minimal-*.map 
+  tar rf $tarball --owner=0 --group=0 --sort=name \
+    --exclude texmf-dist/doc \
+    texmf-dist \
+    || exit 1
   ;;
   docs)
-  # cleanup extracted metafonts/manpages
-  rm -rf texmf-dist
-  echo "Removing manpages/metafonts from -extra/-fonts/-docs which no reside in -base" 
-  tar -f $tarball --delete $(paste $manpages $metafonts) 2>/dev/null
-  # content info, this edition conains all docs from -base
-  mkdir texmf-dist
+  # add docs splittet from base from special packages, add packages index
+  # content info, this edition contains all docs from -base
   echo "Content of -$edition, documentation for -base:" > $texmf/texmf-dist/packages.$edition
   sort $packages_base.doc >> $texmf/texmf-dist/packages.$edition
   gzip -9 $texmf/texmf-dist/packages.$edition
-  tar rf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
+  tar rf $tarball --owner=0 --group=0 --sort=name \
+    texmf-dist/doc/ \
+    texmf-dist/packages.$edition.gz \
+    || exit 1
+  echo "Removing manpages from $edition which now reside in -base" 
+  tar f $tarball --delete $(paste $manpages) || exit 1
   ;;
 esac
+  
+rm -rf texmf-dist
 
 # compress the tarball as everything is in place now
 echo "Compressing $tarball ..."
-- 
cgit v1.2.3