% \iffalse meta-comment
%
% Copyright 2019-2020
% The LaTeX Project and any individual authors listed elsewhere
% in this file.
%
% This file is part of the LaTeX base system.
% -------------------------------------------
%
% It may be distributed and/or modified under the
% conditions of the LaTeX Project Public License, either version 1.3c
% of this license or (at your option) any later version.
% The latest version of this license is in
%    https://www.latex-project.org/lppl.txt
% and version 1.3c or later is part of all distributions of LaTeX
% version 2008 or later.
%
% This file has the LPPL maintenance status "maintained".
%
% The list of all files belonging to the LaTeX base distribution is
% given in the file `manifest.txt'. See also `legal.txt' for additional
% information.
%
% The list of derived (unpacked) files belonging to the distribution
% and covered by LPPL is defined by the unpacking scripts (with
% extension .ins) which are part of the distribution.
%
% \fi
% Filename: ltnews33.tex
%
% This is issue 33 of LaTeX News.

\NeedsTeXFormat{LaTeX2e}[2020-02-02]

\documentclass{ltnews}
\usepackage[T1]{fontenc}

\usepackage{lmodern,url,hologo}

\usepackage{csquotes}
\usepackage{multicol}

\providecommand\meta[1]{$\langle$\textrm{\itshape#1}$\rangle$}
\providecommand\option[1]{\texttt{#1}}
\providecommand\env[1]{\texttt{#1}}
\providecommand\Arg[1]{\texttt\{\meta{#1}\texttt\}}


\providecommand\eTeX{\hologo{eTeX}}
\providecommand\XeTeX{\hologo{XeTeX}}
\providecommand\LuaTeX{\hologo{LuaTeX}}
\providecommand\pdfTeX{\hologo{pdfTeX}}
\providecommand\MiKTeX{\hologo{MiKTeX}}
\providecommand\CTAN{\textsc{ctan}}
\providecommand\TL{\TeX\,Live}
\providecommand\githubissue[2][]{\ifhmode\unskip\fi
     \quad\penalty500\strut\nobreak\hfill
     \mbox{\small\slshape(%
       \href{https://github.com/latex3/latex2e/issues/\getfirstgithubissue#2 \relax}%
          	    {github issue#1 #2}%
           )}%
     \par\smallskip}

% simple solution right now (just link to the first issue if there are more)
\def\getfirstgithubissue#1 #2\relax{#1}

\providecommand\sxissue[1]{\ifhmode\unskip\fi
     \quad\penalty500\strut\nobreak\hfill
     \mbox{\small\slshape(\url{https://tex.stackexchange.com/#1})}\par}

\providecommand\gnatsissue[2]{\ifhmode\unskip\fi
     \quad\penalty500\strut\nobreak\hfill
     \mbox{\small\slshape(%
       \href{https://www.latex-project.org/cgi-bin/ltxbugs2html?pr=#1\%2F#2}%
          	    {gnats issue #1/#2}%
           )}%
     \par}

\let\cls\pkg
\providecommand\env[1]{\texttt{#1}}
\providecommand\acro[1]{\textsc{#1}}

\vbadness=1400  % accept slightly empty columns


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\providecommand\tubcommand[1]{}
\tubcommand{\input{tubltmac}}

\publicationmonth{May}
\publicationyear{2021}

\publicationissue{33}

\begin{document}

\tubcommand{\addtolength\textheight{4.2pc}}   % only for TUB

\maketitle
{\hyphenpenalty=10000 \spaceskip=3.33pt \hbadness=10000 \tableofcontents}

\setlength\rightskip{0pt plus 3em}


\medskip


\section{Introduction}

\emph{to be written}




\section{Extending the hook concept to paragraphs}

Largely triggered by the need for better control of paragraph text
processing, in particular when producing tagged PDF output (see
\cite{33:blueprint}), we have extended the paragraph processing of
\LaTeX{} so that the kernel gains control both at the start and the
end of each paragraph. This is done in a manner that is (or should
be) transparent to packages and user documents.

Beside the internal control points for exclusive use of the \LaTeX{}
kernel we also implemented four public hooks that can be used by
packages or user via the hook management declarations to achieve
special effects or implement manipulations that in the past were only
possible through redefinitions of \cs{everypar} or \cs{par} with the
usual issue that such changes would conflict with changes in other
packages.

The documentation of the hooks together with a few examples is
provided in \file{ltpara-doc.pdf} and for those who want to study the
(quite interesting) code is found in \file{ltpara-code.pdf}. At
some point in the future it will get fully merged into the \LaTeX{}
kernel and will then appear in \file{source2e.pdf}.



\section{Other changes to the \LaTeX{} kernel}

\subsection{Adjusting \env{itemize} labels with \cs{labelitemfont}}

The command \cs{labelitemfont} was in fact already introduced with the
\LaTeX\ release 2020-02-02, but back then we forgot to describe it, so
we do this now. Its purpose is to resolve some bad formatting issues
with the \env{itemize} environment and at the same time make it easier
to adjust its layout if necessary. What could happen in the past was the the
\env{itemize} labels, e.g., the \textbullet{}, would sometimes react to
surrounding font changes and could suddenly change shape, for example
to \textit{\textbullet}.

Now \cs{labelitemfont} is applied to each
label defaulting to \cs{normalfont} which will prevent this behavior.
By choosing a different settings other effects can be achieved, for example
\begin{verbatim}
  \renewcommand\labelitemfont
     {\normalfont\fontfamily{lmss}\selectfont}
  \renewcommand\labelitemfont
     {\rmfamily\normalshape}
\end{verbatim}
The first will take the symbols from Latin Modern Sans so that you get
%
\def\myfont#1{{\let\labelitemfont\empty\fontfamily{lmss}\selectfont#1}}
%
\myfont\labelitemi, \myfont\labelitemii, \myfont\labelitemiii\ and
\myfont\labelitemiv, while the second variant freezes the font family
and shape, but leave the series variable, so that an \env{itemize} in
a bold context would show bolder symbols. Making it empty would give
you the buggy old behavior back.
%
\githubissue{497}


\subsection{A note on file names with spaces, dots or UTF-8 characters}

In one of the the recent \LaTeX{} releases we improved the interface
for specifying file names so that they can now safely contain spaces
(as is common on Windows but also elsewhere), UTF-8 characters
outside the \acro{ascii} range as well as names with several dots in
it. In the past this was only possible by applying a special syntax
(in cases of spaces), not at all for most UTF-8 characters and
not in all circumstances for files with several dots.

However, \TeX{} has a built-in rule saying that you can leave out the
extension if it is \texttt{.tex}. Because of that \verb=\input{file}=
or \verb=\input{file.tex}= both load \file{file.tex} if it
exists. While this is convenient most of the time it is a little
awkward in some scenarios (for example, when both \file{file} and
\file{file.tex} exist) and also when you manually try to implement
that rule.

\LaTeX{} therefore had one special syntax for \cs{include} and
\cs{includeonly}: they always expected that their arguments contains a
file name\footnote{In case of \cs{includeonly} a comma separated list
  of such names.} without its extension, which had to be
\texttt{.tex}.  Thus when you mistakenly wrote
\verb=\include{mychap.tex}= (for example, when you changed from \cs{input}
to \cs{include} somewhere), \LaTeX{} went ahead and looked for the
file \file{mychap.tex.tex} for inclusion and tried to write support
information to the file \file{mychap.tex.aux}.  The reason was that
\cs{include} had to construct both physical file names from the
argument and it didn't bother to do something special about the
extension \texttt{.tex}.

As a side effect of the new implementation this has now changed and
the argument of \cs{include} now gets the extension \texttt{.tex}
removed if it was used. Thus \verb=\include{mychap.tex}= now loads
\file{mychap.tex} and no longer looks for \file{mychap.tex.tex}.
%
\githubissue{486}


\subsection{\cs{end}\texttt{\textbraceleft document\textbraceright}
  should always start in v-mode}

Until now \verb=\end{document}= executed the code from the
\cs{AtEndDocument} hook as its first action. This meant that it was
executed in horizontal mode if the user left no empty line after the
last paragraph.  As a result one could get a spurious space added, for
example, when that code contained a \cs{write} statement. This was
fixed and now \cs{enddocument} first issues a \cs{par} to ensure that
it always starts out in vertical mode.
%
\githubissue{385}



\subsection{Allow extra space between name and address in \pkg{letter} class}

The \cs{opening} command in the the \pkg{letter} class expects the
name and address to be separated by \verb=\\= but it didn't allow to
use an optional argument at this point to add some extra space after
the name. The coding has now been slightly altered to allow for this.
%
\githubissue{427}


\subsection{Add a Lua callback to \pkg{ltshipout} to provide
  a uniform location for applying custom attributes}

Just before shipping out a page, a new \LuaTeX{} callback
\texttt{pre\_shipout\_filter} is now called to allow final
adjustments to the box to be shipped out. This is particularly for
Lua\TeX\ packages which flag certain elements of the page (e.g. using
attributes or properties) in order to apply certain effects to these
elements at shipout. An example for this is the \pkg{luacolor}
package which could insert the color commands using this callback.


\subsection{Improved copy\&paste support for \pdfTeX{} documents}

When compiling with \pdfTeX{}, additional information is added to the
PDF file to improve copying from and searching in text. This especially
allows ligatures to copy correctly from \pdfTeX{} generated PDF files in
most cases.

Since this has been integrated into the kernel, most documents should no
longer need to load the \pkg{cmap} package or input \texttt{glyphtounicode}.
%
\githubissue{465}


\subsection{Provide a hook in \cs{selectfont}}

After \cs{selectfont} has altered the font we run a hook so that
packages can make final adjustments. This functionality was originally
provided by the \pkg{everysel} package, the new implementation is
slightly different and uses the standard hook management.
%
\githubissue{444}


\subsection{Delay change of font series and shape to \cs{selectfont} call}

With the NFSS extensions introduced in 2020 the font series and shape
settings be be influenced by changes to the font family. The setting
is therefore delayed until \cs{selectfont} is executed to avoid
unnecessary or incorrect substitutions that may otherwise happen due
to the order of declarations.
%
\githubissue{444}



\subsection{Allow \cs{nocite} in preamble}

A natural place for \verb=\nocite{*}= would be the preamble of the
document, but for historical reasons \LaTeX{} issued an error message
if it was placed there. From the new release on it is now allowed in
the preamble.
%
\githubissue{424}



\subsection{Shipping out a page while bypassing hooks}

In the 2020 October release several hooks were added to the page
shipout process, e.g., to add some background or foreground material
to some or all pages. We now also added a \cs{RawShipout} command that
bypasses most of these hooks during the shipout. Some essential
internal bookkeeping still takes place such as updating the
\texttt{totalpages} counter or adding \texttt{shipout/firstpage} or
\texttt{shipout/lastpage} material if the page happens to be the first
or last.

\subsection{Robust commands in filename arguments}
The filename handling has been modified so that \verb|\string| is
applied while normalizing robust commands while determining the file
name.  Previously \verb|\input{\sqrt{2}}| would cause \LaTeX\ to loop indefinitely.
With the new behavior it accesses \verb|sqrt {2}.tex|.
%
\githubissue{481}


\subsection{Additional support for Unicode characters from the
  Latin Extended Additional block}

\LaTeX\ is quite capable of typesetting characters such as
\enquote{\d{m}}, but until now it lacked the Unicode mappings for some
characters that are used to write Sanskrit words in Latin
transliteration (as seen in books about yoga, Buddhist philosophy,
etc.). These have now been added so that such characters can be
entered directly instead of resorting to \verb=\d{m}= and so forth.
%
\githubissue{484}


\subsection{Always have color groups set up}

To use color in \LaTeX{} certain constructs, especially boxes, need an
extra layer of groups to ensure that the color setting does not
\emph{escape} and continue outside the box when it shouldn't. To
arrange for this the \LaTeX{} kernel defined a number of commands, e.g.,
\cs{color@begingroup} to be used in such places. They have been
initially no-ops and only the color packages redefined them to become
real groups. This arrangement complicates the coding as one has to
account for a group being there (or not there) depending of what is
loaded in the document. So now the kernel already adds the groups.
%
\githubissue{488}


\subsection{Execute \cs{par} at the end of \cs{marginpar} arguments}

In preparation for tagged PDF it is important to properly tag all
paragraphs and this requires running code at the beginning and end of
each. At the end of a paragraph this is done inside the \cs{par}
command, but the way \cs{marginpar} was coded, \LaTeX{} ended the
marginal note without ever explicitly calling \cs{par}. This has now
been changed.
%
\githubissue{489}



\subsection{Producing several footnote marks to one footnote}

It is sometimes necessary to reference the same footnote several
times, i.e., produce several footnote marks with the same number or
symbol. This is now always possible by placing a \cs{label} into the
\cs{footnote} and reference it with the command \cs{footref}
elsewhere.  This way marks refering to footnotes anywhere on the page
(including those in \texttt{minipage}s) can be generated.  In the past
this command was only available with certain classes or when loading
the \pkg{footmisc} package.
%
\githubissue{482}



\subsection{Providing the raw option list of packages or documentclass to key/value handlers}

\LaTeXe{} has always normalized space in option lists so\\
\verb|\documentclass[ a4paper , 12pt ]{article}|\\
processed the intended options \texttt{a4paper} and \texttt{12pt}.

Unfortunately the mechanism used was designed for the simple option
names of the standard option processing.  Many classes and packages 
now use extended \emph{keyval} processing, however this white space
normalisation makes this difficult:
\verb|[bb=1 2 3 4]|
which might be expected to pass a bounding box of four numbers is
normalised to \texttt{bb=1234} and
\verb|[bb={1 2 3 4}]|\\
which might be expected to quote the spaces results in low level \TeX{}
parsing errors.


For compatibility reasons, the standard option processing has not been
changed however the original un-normalised package and class option lists
are now saved. They are not used in the standard processing, however
extended package option systems may use these \enquote{raw} option list
macros if they are defined.

The one change affecting the standard processing is that the low level
error mentioned above is now avoided as values (any tokens to the
right of an =  sign) are removed from consideration from the \enquote{unused
option list}.  In this release \texttt{clip=true} and
\texttt{clip=false} both contribute \texttt{clip} to the list of
options that have been used.
%
\githubissue{85}



\subsection{Poor man's \cs{textasteriskcentered} if missing}

The \cs{textasteriskcentered} symbol, used as part of the set of
footnote symbols in \LaTeX{}, is assumed to be implemented by
every font in the \texttt{TS1} encoding (when \pdfTeX{} is used) or
in the \texttt{TU} encoding for the Unicode engines. Unfortunately,
that assumption is not correct for all fonts, for example, for the
\texttt{stix2} fonts don't offer the glyph, with the result that one
gets missing glyphs when using \cs{thanks} etc.

For that reason the definition for \cs{textasteriskcentered} was
altered to check if there is a glyph in the right position and if not
a normal \enquote{*} is used, slightly enlarged and lowered.  That may
not be perfect in all cases, but certainly better than nothing show
up.
%
\githubissue{502}



\subsection{Provide more ``dashes'' in encodings \texttt{OT1}, \texttt{T1} and \texttt{TU}}

When pasting in text from external sources one sometimes encounters the Unicode characters
%
\texttt{"2011} (non-breaking hyphen),
\texttt{"2012} (figure dash) and
\texttt{"2015} (horizontal bar)
%
in addition to the common \texttt{"2013} (en-dash) and \texttt{"2014}
(em-dash). In the past the first three characters produced an error
message when used with \pdfTeX{}. Now they typeset an approximation
(as they are unavailable in \texttt{OT1} or \texttt{T1} encoded fonts
used by \pdfTeX{}), e.g., the figure dash is approximated by an en-dash.

In Unicode engines they either work (if contained in the selected
Unicode font) or typeset nothing and produce a ``Missing character''
warning in the log file.

However, what works in all engines now, is to access the characters
via the command names \cs{textnonbreakinghyphen}, \cs{textfiguredash}
and \cs{texthorizontalbar}, respectively.
%
\githubissue{404}



\subsection{\ldots}

\emph{to be written}
%
\githubissue{xxx}



\section{Changes to packages in the \pkg{graphics} category}

\subsection{\ldots}

%
\githubissue{xxx}



\section{Changes to packages in the \pkg{tools} category}

\subsection{\pkg{layout}: Support Japanese as a language option}
The package now recognizes \texttt{japanese} as a language option.
%
\githubissue{353}



\subsection{\pkg{longtable}: General bug fix update}
Minor update to \pkg{longtable} to fix bugs reported.  Notably the
possibility of incorrect page breaks if floats appear on the same page
that a \env{longtable} starts.  As this may affect page breaking in
existing documents, a rollback to \pkg{longtable 4.13}
(\file{longtable-2020-01-07.sty}) is supported.
%
\gnatsissue{tools}{3512}





\section{Changes to packages in the \pkg{amsmath} category}

\subsection{\ldots}

%
\githubissue{xxx}


\medskip
\begin{thebibliography}{9}

\fontsize{9.3}{11.3}\selectfont

\bibitem{33:blueprint} Frank Mittelbach and Chris Rowley:
  \emph{\LaTeX{} Tagged PDF — A blueprint for a large project}.\\
  \url{https://latex-project.org/publications/indexbyyear/2020/}

\end{thebibliography}



\end{document}


\endinput % needed for ltnews processing

\bibitem{32:ltnews31} \LaTeX{} Project Team:
  \emph{\LaTeXe{} news 31}.\\
  \url{https://latex-project.org/news/latex2e-news/ltnews31.pdf}

\bibitem{32:site-doc}
  \emph{\LaTeX{} documentation on the \LaTeX{} Project Website}.\\
  \url{https://latex-project.org/help/documentation/}

\bibitem{32:issue-tracker}
  \emph{\LaTeX{} issue tracker}.
  \url{https://github.com/latex3/latex2e/issues/}

\bibitem{32:babel}
  Javier Bezos and Johannes Braams.
  \emph{Babel---Localization and internationalization}.\\
  \url{https://www.ctan.org/pkg/babel}


  
