% !Rnw weave = knitr
%%% DO NOT EDIT the .tex file directly since it is generated from the .Rnw
%%% sources.
%\VignetteEngine{knitr::knitr}
%\VignetteIndexEntry{irace package: User Guide}
%\VignetteDepends{knitr}
%\VignetteCompiler{knitr}
\synctex=1
\RequirePackage{xparse}
\RequirePackage[dvipsnames]{xcolor}
\documentclass[a4paper,english]{article}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{lmodern}
\usepackage[a4paper]{geometry} % It saves some pages
\usepackage[english]{babel}
\usepackage{ifthen}
\newboolean{Release}
\setboolean{Release}{true}
\usepackage{calc}
\usepackage{afterpage}
\usepackage{algorithm,algorithmic}
\usepackage{booktabs}
\usepackage{tabularx}
\usepackage{xspace}
\usepackage{amsmath,amssymb}
\usepackage{relsize}
\usepackage{fancyvrb}
\usepackage{underscore}
\usepackage{microtype}
% \texttt{test -- test} keeps the "--" as "--" (and does not convert it to an en dash)
\DisableLigatures{encoding = T1, family = tt* }
\usepackage[hyphens]{url}
\usepackage{hyperref}
\usepackage[numbers]{natbib}
\usepackage[nottoc]{tocbibind}
%% For autoref
\hypersetup{
colorlinks,
linkcolor={red!50!black},
citecolor={blue!50!black},
urlcolor={blue!70!black}
}
\addto\extrasenglish{%
\def\sectionautorefname{Section}
\let\subsectionautorefname\sectionautorefname
\let\subsubsectionautorefname\sectionautorefname
}
\usepackage[titletoc, title]{appendix}
% Fix use with \autoref
\newcommand*{\Appendixautorefname}{Appendix}
\usepackage{tocloft}
\setlength{\cftsubsecnumwidth}{3em}% Set length of number width in ToC for \subsection
\usepackage[inline]{enumitem}
\setlist[enumerate]{leftmargin=*,widest=00}
\setlist[itemize]{leftmargin=1.5em}
%% FIXME: listing is very limited, we should use 'minted'
\usepackage{listings}
\lstdefinestyle{BashInputStyle}{
language=bash,%
basicstyle=\ttfamily,%
numbers=none,%
frame=tb,%
rulecolor=\color{lightgray},
% framesep=1ex,
framexleftmargin=1ex,
columns=fullflexible,%
backgroundcolor=\color{yellow!05},%
linewidth=\linewidth,%
% xleftmargin=1\linewidth,%
identifierstyle=\color{darkgray},%
keywordstyle=\color{darkgray},%
keywordstyle={[2]\color{Cyan}},%
keywordstyle={[3]\color{olive}},%
stringstyle=\color{MidnightBlue},%
commentstyle=\color{RedOrange},%
morestring=[b]',%
showstringspaces=false
}
\DefineVerbatimEnvironment{Code}{Verbatim}{}
\DefineVerbatimEnvironment{CodeInput}{Verbatim}{fontshape=rm}
\DefineVerbatimEnvironment{CodeOutput}{Verbatim}{}
\newenvironment{CodeChunk}{}{}
\newcommand{\IRACEHOME}[1]{\hyperlink{irace_home}{\path{$IRACE_HOME}}\path{#1}}
\providecommand{\keywords}[1]{\textbf{\textit{Index terms---}} #1}
% Simple font selection is not good enough. For example, |\texttt{--}|
% gives `\texttt{--}', i.e., an endash in typewriter font. Hence, we
% need to turn off ligatures, which currently only happens for commands
% |\code| and |\samp| and the ones derived from them. Hyphenation is
% another issue; it should really be turned off inside |\samp|. And
% most importantly, \LaTeX{} special characters are a nightmare. E.g.,
% one needs |\~{}| to produce a tilde in a file name marked by |\file|.
% Perhaps a few years ago, most users would have agreed that this may be
% unfortunate but should not be changed to ensure consistency. But with
% the advent of the WWW and the need for getting `|~|' and `|#|' into
% URLs, commands which only treat the escape and grouping characters
% specially have gained acceptance
\makeatletter
\DeclareRobustCommand\code{\bgroup\@makeother\_\@makeother\~\@makeother\$\@noligs\@codex}
\def\@codex#1{\texorpdfstring%
{{\text{\normalfont\ttfamily\hyphenchar\font=-1 #1}}}%
{#1}\egroup}
\makeatother
\let\proglang=\textsf
\newcommand{\pkg}[1]{{\fontseries{b}\selectfont #1}}
\newcommand{\aR}{\proglang{R}\xspace}
\newcommand{\MATLAB}{\proglang{MATLAB}\xspace}
\newcommand{\eg}{e.g.,\xspace}
\newcommand{\SoftwarePackage}{\pkg}
\newcommand{\ACOTSP}{\SoftwarePackage{ACOTSP}\xspace}
%% How to use this command:
% Parameter with one short switch: \defparameter[short]{paramName}{long}{default}
% Parameter without short switch: \defparameter{paramName}{long}{default}
% Parameter without switch: \defparameter{paramName}{}{default}
\newcommand{\defparameter}[4][]{%
\item[\code{#2}]\hypertarget{opt:#2}{} ~~ %
\ifthenelse{\equal{#3}{}}{}{%
\emph{flag:} %
\ifthenelse{\equal{#1}{}}{}{%
\code{-#1}~~~\emph{or}~~~}%
\code{--#3} ~~ }%
\emph{default:}~\texttt{#4} \\
}
\newcommand{\parameter}[1]{\hyperlink{opt:#1}{\code{#1}}}
\newcommand{\iracefun}[1]{\href{https://mlopez-ibanez.github.io/irace/reference/#1.html}{\code{#1()}}}
%\usepackage{showlabels}
%\showlabels{hypertarget}
\newcommand{\irace}{\pkg{irace}\xspace}
\newcommand{\Irace}{\pkg{Irace}\xspace}
\newcommand{\race}{\pkg{race}\xspace}
\newcommand{\FRACE}{\text{F-Race}\xspace}
\newcommand{\IFRACE}{\text{I/F-Race}\xspace}
\newcommand{\PyImp}{\pkg{PyImp}\xspace}
\newcommand{\iraceversion}{\Sexpr{packageVersion("irace")}}
\newcommand{\Niter}{\ensuremath{N^\text{iter}}\xspace}
\newcommand{\Nparam}{\ensuremath{{N^\text{param}}}\xspace}
\newcommand{\iter}{\ensuremath{j}\xspace}
\newcommand{\Budget}{\ensuremath{B}\xspace}
\newcommand{\Budgetj}{\ensuremath{\Budget_{\iter}}\xspace}
\newcommand{\Bused}{\ensuremath{\Budget_\text{used}}\xspace}
\newcommand{\Ncand}[1][]{\ensuremath{N_{#1}}\xspace}
\newcommand{\Mui}{\ensuremath{\mu_{\iter}}\xspace}
\newcommand{\Nmin}{\ensuremath{N^\text{min}}\xspace}
\newcommand{\Nsurv}{\ensuremath{N^\text{surv}}\xspace}
\newcommand{\Nelite}{\ensuremath{N^\text{elite}}\xspace}
\newcommand{\Nnew}{\ensuremath{N^\text{new}}\xspace}
\newcommand{\bmax}{\ensuremath{b^\text{max}}\xspace}
\newcommand{\bmin}{\ensuremath{b^\text{min}}\xspace}
\newcommand{\Celite}{\ensuremath{\Theta^\text{elite}}\xspace}
\ifthenelse {\boolean{Release}}{%
\newcommand{\MANUEL}[1]{}
\newcommand{\LESLIE}[1]{}
\newcommand{\THOMAS}[1]{}
}{%
\newcommand{\MANUEL}[1]{{\footnotesize\noindent\textbf{\color{red}[~MANUEL: #1~]}}}
\newcommand{\LESLIE}[1]{\footnote{\noindent\textbf{[ LESLIE: #1 ]}}}
\newcommand{\THOMAS}[1]{\footnote{\noindent\textbf{[ THOMAS: #1 ]}}}
}
\newcommand{\hide}[1]{}
\usepackage{tcolorbox}
\newcommand{\infoicon}{%
\parbox[c]{0.75cm}{\includegraphics[keepaspectratio=true,width=0.75cm]{light-bulb-icon}}%
\hspace{1em}}
\newcommand{\warningicon}{%
\parbox[c]{0.75cm}{\includegraphics[keepaspectratio=true,width=0.75cm]{Warning-icon}}%
\hspace{1em}}
\definecolor{LightGray}{RGB}{193,193,193}
\definecolor{LightYellow}{RGB}{253,247,172}
\newlength\macroiconwidth
\newenvironment{xwarningbox}{%
\setlength{\fboxrule}{3.0\fboxrule}%
\setlength{\fboxsep}{0\fboxsep}%
\begin{tcolorbox}[colback=LightYellow,colframe=LightGray,boxrule=\fboxrule,boxsep=\fboxsep]%
\infoicon%
\settowidth{\macroiconwidth}{\infoicon}%
\begin{minipage}[c]{\columnwidth - \macroiconwidth - 2.0\fboxrule - 2.0\fboxsep}
\raggedright\footnotesize
%
}{%
\end{minipage}
\end{tcolorbox}
%
}
% Workaround for broken knitr: https://github.com/yihui/knitr-examples/blob/master/036-latex-if.tex
\providecommand{\hldef}[1]{\textcolor[rgb]{0.345,0.345,0.345}{#1}}%
\providecommand{\hlsng}[1]{\textcolor[rgb]{0.192,0.494,0.8}{#1}}%
<<setup, include=FALSE>>=
library(knitr)
knit_hooks$set(inline = function(x) {
if (is.numeric(x)) return(knitr:::format_sci(x, 'latex'))
highr::hi_latex(x)
})
@
\begin{document}
<<include=FALSE>>=
library(knitr)
@
\author{Manuel L\'opez-Ib\'a\~nez, Leslie P\'erez C\'aceres, J\'er\'emie Dubois-Lacoste,\\
Thomas St\"utzle and Mauro Birattari
\\IRIDIA, CoDE, Universit\'e Libre de Bruxelles, Brussels, Belgium}
\title{The \irace Package: User Guide}
\date{Version \iraceversion, \today}
%\keywords{automatic
% algorithm configuration, racing, parameter tuning, \aR}
\maketitle
\tableofcontents
%Load files needed for examples
<<exampleload,eval=TRUE,include=FALSE>>=
library("irace")
load("examples.Rdata") # loads "experiment" and "output"
iraceResults <- irace::read_logfile(system.file(package="irace", "exdata", "irace-acotsp.Rdata", mustWork=TRUE))
log_ablation_file <- system.file(package="irace", "exdata", "log-ablation.Rdata", mustWork = TRUE)
load(log_ablation_file)
options(width = 70)
@
\newpage
%%
%%
%%
%% General info
%%
%%
%%
\section{General information}
\MANUEL{Some things could be taken from the intro of the irace paper and reformulated.}
\MANUEL{It would be good to mention that not only opt algorithms can be configured with irace, we say this in the paper.}
\subsection{Background}
\MANUEL{I would add a paragraph defining what is irace (a bit longer than the abstract above) and references to the literature so people can find more info. The first reference should be the irace TR.} \LESLIE{Here i guess we should say why tune an algorithm is a good idea, and why using irace is a better one.}
The \irace package implements an \emph{iterated racing} procedure,
which is an extension of Iterated F-race (\IFRACE)~\cite{BirYuaBal2010:emaoa}. The main use of
\irace is the automatic configuration of optimization and decision algorithms,
that is, finding the most appropriate settings of an algorithm given a
set of instances of a problem. However, it may also be useful for
configuring other types of algorithms when performance depends on
the used parameter settings. It builds upon the \pkg{race} package
by Birattari and it is implemented in \aR. The \irace
package is available from CRAN:
%
\begin{center}
\url{https://cran.r-project.org/package=irace}
\end{center}
%
More information about \irace is available at
\url{https://mlopez-ibanez.github.io/irace}.
\subsection{Version}
The current version of the \irace package is \iraceversion. Previous
versions of the package can also be found in the \href{https://cran.r-project.org/package=irace}{CRAN website}.
The algorithm underlying the current version of \irace and its motivation are
described by \citet{LopDubPerStuBir2016irace}. The \textbf{adaptive capping
mechanism} available from version $3.0$ is described by
\citet{PerLopHooStu2017:lion}. Details of the implementation before version 2.0
can be found in a previous technical report~\cite{LopDubStu2011irace}.
%
\begin{xwarningbox}
Versions of \irace before 2.0 are not compatible with the file formats
detailed in this document.
\end{xwarningbox}
\subsection{License}
The \irace package is Copyright \copyright{} \the\year\ and distributed under the GNU General Public
License version 3.0 (\url{http://www.gnu.org/licenses/gpl-3.0.en.html}).
The \irace package is free software (software libre): You can redistribute it and/or modify it under the terms
of the GNU General Public License as published by the Free Software Foundation,
either version 3 of the License, or (at your option) any later version.
The \irace package is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.
Please be aware that the fact that this program is released as Free Software
does not excuse you from scientific propriety, which obligates you to give
appropriate credit! If you write a scientific paper describing research that
made substantive use of this program, it is your obligation as a scientist to
(a) mention the fashion in which this software was used in the Methods section;
(b) mention the algorithm in the References section. The appropriate citation
is:
\begin{itemize}[leftmargin=3em]
\item[] Manuel López-Ibáñez, Jérémie Dubois-Lacoste, Leslie Pérez Cáceres,
Thomas Stützle, and Mauro Birattari. The \irace package: Iterated Racing for
Automatic Algorithm Configuration. \emph{Operations Research Perspectives}, 3:43--58, 2016.
doi:~\href{http://dx.doi.org/10.1016/j.orp.2016.09.002}{10.1016/j.orp.2016.09.002}
\end{itemize}
\section{Before starting}
\MANUEL{I think this could be a bit more detailed by defining what is a parameter, a configuration, an instance, etc. but ok for now.}
The \irace package provides an automatic configuration tool for
tuning optimization algorithms, that is, automatically finding good configurations for the parameters values
of a (target) algorithm saving the effort that normally requires manual
tuning.
\begin{figure}[t]
\centering
\includegraphics[width=0.6\textwidth]{irace-scheme}
\caption{Scheme of \irace flow of information.}
\label{fig:irace-scheme}
\end{figure}
Figure~\ref{fig:irace-scheme} gives a general scheme of how \irace works.
\Irace receives as input a \emph{parameter space definition} corresponding to the
parameters of the target algorithm that will be tuned, a set of \emph{instances}
for which the parameters must be tuned for and a set of options for \irace that define the \emph{configuration scenario}. Then, \irace searches in the parameter search space for good performing algorithm
configurations by executing the target algorithm on different instances and
with different parameter configurations. A \parameter{targetRunner} must be provided to execute the target algorithm with a
specific parameter configuration ($\theta$) and instance ($i$). The \parameter{targetRunner} function (or program) acts
as an interface between the execution of the target algorithm and \irace: It
receives the instance and configuration as arguments and must return the
evaluation of the execution of the target algorithm.
The following user guide contains guidelines for installing \irace, defining
configuration scenarios, and using \irace to automatically configure your
algorithms.
%%
%%
%%
%% Installation
%%
%%
%%
\section{Installation}
\subsection{System requirements}
\begin{itemize}
\item \aR ($\text{version} \geq 3.2.0$) is required for running irace, but you don't
need to know the \aR language to use it. \aR is freely available and you can
download it from the \aR project website
(\url{https://www.r-project.org}). See \autoref{sec:installation} for a
quick installation guide of \aR.
\item For GNU/Linux and OS X, the command-line executable
\code{parallel-irace} requires GNU Bash. Individual examples may require additional software.
\end{itemize}
\subsection{\irace installation} \label{sec:irace install}
The \irace package can be installed automatically within \aR or
by manual download and installation. We advise to use the automatic
installation unless particular circumstances do not allow it. The instructions
to install \irace with the two mentioned methods are the following:
\subsubsection[Install automatically within R]{Install automatically within \aR{}}
Execute the following line in the \aR console to install the package:
<<R_irace_install, prompt=FALSE, eval=FALSE>>=
install.packages("irace")
@
Select a mirror close to your location, and test the installation in the \aR console with:
<<R_irace_launch,eval=FALSE, prompt=FALSE>>=
library("irace")
q() # To exit R
@
Alternatively, within the \aR graphical interface, you may use
the \code{Packages and data->Package installer} menu on OS X or the \code{Packages} menu on Windows.
\subsubsection{Manual download and installation}
From the \irace package CRAN website
(\url{https://cran.r-project.org/package=irace}), download one of the three
versions available depending on your operating system:
\begin{itemize}
\item \code{irace_\iraceversion.tar.gz} (Unix/BSD/GNU/Linux)
\item \code{irace_\iraceversion.tgz} (OS X)
\item \code{irace_\iraceversion.zip} (Windows)
\end{itemize}
To install the package on GNU/Linux and OS X, you must execute the following
command at the shell (replace \code{<package>} with the path to the downloaded file, either \code{irace_\iraceversion.tar.gz} or \code{irace_\iraceversion.zip}):
%
\begin{lstlisting}[style=BashInputStyle]
R CMD INSTALL <package>
\end{lstlisting}
To install the package on Windows, open \aR and execute the following line on
the \aR console (replace \code{<package>} with the path to the downloaded file \code{irace_\iraceversion.zip}):
%\LESLIE{Check that this actually works on internet says that this: \code{Rscript -e "install.packages('foo.zip', repos = NULL)"} also works}
%
<<install_win1,eval=FALSE, prompt=FALSE>>=
install.packages("<package>", repos = NULL)
@
If the previous installation instructions fail because of insufficient
permissions and you do not have sufficient admin rights to install \irace
system-wide, then you need to force a local installation.
\subsubsection{Local installation}
Let's assume you wish to install \irace on a path denoted by
\code{<R_LIBS_USER>}, which is a filesystem path for which you have sufficient
rights. This directory \textbf{must} exist before attempting the
installation. Moreover, you must provide to \aR the path to this
library when loading the package. However, the latter can be avoided by adding
the path to the system variable \code{R_LIBS} or to the \aR internal variable
\code{.libPaths}, as we will see below.\footnote{%
On Windows, see also
\url{https://cran.r-project.org/bin/windows/base/rw-FAQ.html\#I-don_0027t-have-permission-to-write-to-the-R_002d3_002e3_002e1_005clibrary-directory}.}
On GNU/Linux or OS X, execute the following commands to install the
package on a local directory:
\begin{lstlisting}[style=BashInputStyle]
export R_LIBS_USER="<R_LIBS_USER>"
# Create R_LIBS_USER if it doesn't exist
mkdir $R_LIBS_USER
# Replace <package> with the path to the downloaded file.
R CMD INSTALL --library=$R_LIBS_USER <package>
# Tell R where to find R_LIBS_USER
export R_LIBS=${R_LIBS_USER}:${R_LIBS}
\end{lstlisting}
On Windows, you can install the package on a local directory by executing the
following lines in the \aR console:
<<install_win2,eval=FALSE, prompt=FALSE>>=
# Replace <package> with the path to the downloaded file.
# Replace <R_LIBS_USER> with the path used for installation.
install.packages("<package>", repos = NULL, lib = "<R_LIBS_USER>")
# Tell R where to find R_LIBS_USER.
# This must be executed for every new session.
.libPaths(c("<R_LIBS_USER>", .libPaths()))
@
\subsubsection{Testing the installation and invoking irace}
Once \irace has been installed, load the package and test that the installation
was successful by opening an \aR console and executing:
<<R_irace_test1, prompt=FALSE, eval=FALSE>>=
# Load the package
library("irace")
# Obtain the installation path
system.file(package = "irace")
@
The last command must print out the filesystem path where \irace is
installed. In the remainder of this guide, the variable
\code{\$IRACE_HOME} is used to denote this path. When executing any provided
command that includes the \code{\$IRACE_HOME} variable do not forget to replace
this variable with the installation path of \irace.
On GNU/Linux or OS X, you can let the operating system know where to find \irace
by defining the \code{\$IRACE_HOME} variable and adding it to the system
\code{PATH}. Append the following commands to \path{~/.bash_profile}, \path{~/.bashrc} or
\path{~/.profile}:
%
%<<linux_irace_path1,engine='bash',eval=FALSE>>=
\begin{lstlisting}[style=BashInputStyle]
# Replace <IRACE_HOME> with the irace installation path
export IRACE_HOME=<IRACE_HOME>
export PATH=${IRACE_HOME}/bin/:$PATH
# Tell R where to find R_LIBS_USER
# Use the following line only if local installation was forced
export R_LIBS=${R_LIBS_USER}:${R_LIBS}
\end{lstlisting}
%@
Then, open a new terminal and launch \irace as follows:
%<<linux_irace_help,engine='bash',eval=FALSE>>=
\begin{lstlisting}[style=BashInputStyle]
irace --help
\end{lstlisting}
%@
On Windows, you need to add both \aR and the installation path of \irace to the
environment variable \code{PATH}. To edit the \code{PATH}, search for
``Environment variables'' in the control panel, edit \code{PATH} and add
a string similar to \path{C:\R_PATH\bin;C:\IRACE_HOME\bin\x64\} where \code{R_PATH}
is the installation path of \aR and \code{IRACE_HOME} is the installation path
of \irace. If \irace was installed locally, you also need to edit the
environment variable \code{R_LIBS} to add \code{R_LIBS_USER}. Then, open a new
terminal (run program \code{cmd.exe}) and launch \irace as:
%
%<<win_irace_help,engine='bash',eval=FALSE>>=
\begin{lstlisting}[style=BashInputStyle]
irace.exe --help
\end{lstlisting}
%@
Alternatively, you may directly invoke \irace from within the \aR
console by executing:
<<windows_irace_help ,eval=FALSE, prompt=FALSE>>=
library("irace")
irace_cmdline("--help")
@
\section{Running irace}\label{sec:execution}
Before performing the tuning of your algorithm, it is necessary to define a
tuning scenario that will give \irace all the necessary information to optimize the
parameters of the algorithm. The tuning scenario is composed of the following elements:
\begin{enumerate}
\item Target algorithm parameter description (see \autoref{sec:target parameters}).
\item Target algorithm runner (see \autoref{sec:runner}).
\item Training instances list (see \autoref{sec:training})
\item \irace options (see \autoref{sec:irace options}).
\item \textit{Optional:} Initial configurations (see \autoref{sec:initial}).
\item \textit{Optional:} Target algorithm evaluator (see \autoref{sec:evaluator}).
\end{enumerate}
These scenario elements can be provided as plain text files
or as \aR objects. This user guide provides examples of both types,
but we advise the use of plain text files, which we consider the simpler
option.
For a step-by-step guide to create the scenario elements for your target algorithm
continue to \autoref{sec:step}. For an example execution of \irace using the
\ACOTSP scenario go to \autoref{sec:example}.
\subsection{Step-by-step setup guide}\label{sec:step}
This section provides a guide to setup a basic execution of \irace. The
template files provided in the package (\IRACEHOME{/templates}) will be
used as basis for creating your new scenario. Please follow carefully the
indications provided in each step and in the template files used; if you have
doubts check the the sections that describe each option in detail.
\begin{enumerate}[leftmargin=*]
\item Create a directory (\eg~\path{./tuning/}) for the scenario setup. This directory will contain
all the files that describe the scenario. On GNU/Linux or OS X, you can do this as follows:
%<<dir1,engine='bash',eval=FALSE>>=
\begin{lstlisting}[style=BashInputStyle]
mkdir ./tuning
cd ./tuning
\end{lstlisting}
%@
\item Initialize the tuning directory with template config files. On GNU/Linux or OS X, you can do this as follows:
%<<dir1,engine='bash',eval=FALSE>>=
\begin{lstlisting}[style=BashInputStyle]
irace --init
\end{lstlisting}
%@
\item Define the target algorithm parameters to be tuned by following the
instructions in \code{parameters.txt}. Available parameter types and other
guidelines can be found in \autoref{sec:target parameters}.
\item \textit{Optional}: Define the initial parameter configuration(s) of your
algorithm, which allows you to provide good starting configurations (if you know
some) for the tuning. Follow the instructions in \code{configurations.txt} and set \parameter{configurationsFile}\code{="configurations.txt"} in \code{scenario.txt}. More
information in \autoref{sec:initial}. If you do not need to
define initial configurations remove this file from the directory.
\item Place the instances you would like to use for the tuning of your
algorithm in the folder \path{./tuning/Instances/}. In addition, you can
create a file (\eg~\code{instances-list.txt}) that specifies which instances
from that directory should be run and which instance-specific parameters to
use. To use such an instance file, set the appropriate option in
\code{scenario.txt}, e.g., \parameter{trainInstancesFile} \code{= "instances-list.txt"}.
See \autoref{sec:training} for guidelines.
\item Uncomment and assign in \code{scenario.txt} only the options for which you
need a value different from the default. Some common options that you might want to adjust are:
\begin{description}
\item[\parameter{execDir}] (\code{--exec-dir}): the directory in which \irace will execute the
target algorithm; the default value is the current directory.
%\item[\parameter{logFile}] (\code{--log-file}): a file the name of the results \aR data file that produces \irace.
\item[\parameter{maxExperiments}] (\code{--max-experiments}): the maximum number of executions
of the target algorithm that \irace will perform.
\item[\parameter{maxTime}] (\code{--max-time}): maximum total execution time in seconds for the executions of \code{targetRunner}. In this case, \code{targetRunner} must return two values: cost and time. Note that you must provide either \parameter{maxTime} or \parameter{maxExperiments}.
\item[\parameter{trainInstancesDir}] (\code{--train-instances-dir}): set to \path{./Instances} if you put the training instances in that folder as instructed above.
\end{description}
For setting the tuning budget, see \autoref{sec:budget}. For more information on \irace options and their default values, see \autoref{sec:irace options}.
\item Modify the \code{target-runner} script to run your algorithm. This script
must execute your algorithm with the parameters and instance specified by
\irace and return the evaluation of the execution and \textit{optionally} the execution
time (\code{cost [time]}). When the \parameter{maxTime} option is used, returning \code{time} is mandatory.
The \code{target-runner} template is written in \proglang{GNU Bash}
scripting language, which can be executed easily in GNU/Linux and OS X
systems. However, you may use any other programming language. We provide examples written in \proglang{Python}, \proglang{MATLAB} and other languages in \IRACEHOME{/examples/}. An example using Julia is available at \url{https://github.com/sbomsdorf/An-example-of-irace-using-Julia-code}. %
Follow these instructions to adjust the given \code{target-runner} template
to your algorithm:
\begin{enumerate}
\item Set the \code{EXE} variable with the path to the executable of the target algorithm.
\item Set the \code{FIXED_PARAMS} if you need extra arguments in the
execution line of your algorithm. An example could be the time that your
algorithm is required to run
(\code{FIXED_PARAMS}\hspace{0pt}\code{=}\hspace{0pt}\code{"--time 60"}) or
the number of evaluations required (\code{FIXED_PARAMS}\hspace{0pt}\code{=}\hspace{0pt}\code{"--evaluations 10000"}).
\item The line provided in the template executes the executable described in the \code{EXE} variable.
%
\begin{center}
\code{\$EXE \$\{FIXED_PARAMS\} -i \$\{INSTANCE\} --seed \$\{SEED\} \$\{CONFIG_PARAMS\}}
\end{center}
%
You must change this line according to the way your algorithm is executed. In this
example, the algorithm receives the instance to solve with the flag
\code{-i} and the seed of the random number generator with the flag \code{--seed}. The variable
\code{CONFIG_PARAMS} adds to the command line the parameters that \irace has
given for the execution. You must set the command line execution as
needed. For example, the instance might not need a flag and might need to be
the first argument:
\begin{center}
\code{\$EXE \$\{INSTANCE\} \$\{FIXED_PARAMS\} --seed \$\{SEED\} \$\{CONFIG_PARAMS\}}
\end{center}
The output of your algorithm is saved to the file defined in the
\code{\$STDOUT} variable, and error output is saved in the file given by
\code{\$STDERR}. The line:
\begin{center}
\code{if [ -s "\${STDOUT}" ]; then}
\end{center}
checks if the file containing the output of your algorithm is not
empty. The example provided in the template assumes that your algorithm
prints in the last output line the best result found (only a number). The
line:
\begin{center}
\code{COST=\$(cat \$\{STDOUT\} | grep -e '\^{}[[:space:]]*[+-]\textbackslash{}?[0-9]' | cut -f1)}
\end{center}
parses the output of your algorithm to obtain the result from the last line. The \code{target-runner}
script must print \textbf{only} one number. In the template example, the result is printed with
\code{echo "\$COST"} (assuming \parameter{maxExperiments} is used) and the generated files are deleted (you may remove that line if you wish to keep them).
\begin{xwarningbox}
The \code{target-runner} script must be an executable file, unless you specify \parameter{targetRunnerLauncher}.
\end{xwarningbox}
You can test the target runner from the \aR console by checking the scenario as
explained earlier in \autoref{sec:execution}.
If you have problems related to the \code{target-runner} script when executing
\irace, see \autoref{sec:check list} for a check list to help diagnose common
problems. For more information about the \parameter{targetRunner}, please see
\autoref{sec:runner},
\end{enumerate}
\item \textit{Optional}: Modify the \code{target-evaluator} file. This is rarely needed and the \code{target-runner} template does not use it. \autoref{sec:evaluator} explains when a \parameter{targetEvaluator} is needed and how to define it.
\item The \irace executable provides an option (\parameter{--check}) to
check that the scenario is correctly defined. We recommend to perform a
check every time you create a new scenario. When performing the check,
\irace will verify that the scenario and parameter definitions are
correct and will test the execution of the target algorithm. To check
your scenario execute the following commands:
\begin{itemize}
\item From the command-line (on Windows, execute \code{irace.bat}):
%<<irace_check,engine='bash', eval=FALSE>>=
\begin{lstlisting}[style=BashInputStyle]
# $IRACE_HOME is the installation directory of irace.
$IRACE_HOME/bin/irace --scenario scenario.txt --check
\end{lstlisting}
%@
\item Or from the \aR console:
<<irace_R_check, eval=FALSE, prompt=FALSE>>=
library("irace")
scenario <- readScenario(filename = "scenario.txt",
scenario = defaultScenario())
checkIraceScenario(scenario = scenario)
@
\end{itemize}
\item Once all the scenario elements are prepared you can execute \irace, either using the command-line wrappers provided by the package
or directly from the \aR console:
\begin{itemize}
\item
\textbf{From the command-line console}, call the command (on Windows, you should execute
\code{irace.exe}):
\begin{lstlisting}[style=BashInputStyle]
cd ./tuning/
# $IRACE_HOME is the installation directory of irace
# By default, irace reads scenario.txt, you can specify a different file
# with --scenario.
$IRACE_HOME/bin/irace
\end{lstlisting}
For this example, we assume that the needed scenario files have been set
properly in the \code{scenario.txt} file using the options described in
\autoref{sec:irace options}. Most \irace options can be specified
in the command line or directly in the \code{scenario.txt} file.
\item
\textbf{From the \aR console}, evaluate:
<<irace_R_exe, eval=FALSE, prompt=FALSE>>=
library("irace")
# Go to the directory containing the scenario files
setwd("./tuning")
scenario <- readScenario(filename = "scenario.txt",
scenario = defaultScenario())
irace_main(scenario = scenario)
@
\end{itemize}
This will perform one run of \irace. See the output of \code{irace --help} in the command-line or \code{irace_cmdline("--help")} in \aR for quick information on
additional \irace options. For more information about \irace options, see \autoref{sec:irace options}.
\end{enumerate}
\begin{xwarningbox}
Command-line options override the same options specified in the \code{scenario.txt} file.
\end{xwarningbox}
\subsection{Setup example for ACOTSP}\label{sec:example}
The \ACOTSP tuning example can be found in the package installation in the folder \IRACEHOME{/examples/acotsp}.
%
Other example scenarios can be found in the same folder. More
examples of tuning scenarios can be found in the Algorithm Configuration Library (AClib, \url{http://www.aclib.net/}).
In this section, we describe how to execute the \ACOTSP scenario. If you wish to start setting up
your own scenario, continue to the next section. For this example, we assume
a GNU/Linux system such as Ubuntu with a working \proglang{C} compiler such as \code{gcc}.
To execute this scenario follow these steps:
\begin{enumerate}
\item Create a directory for the tuning (\eg~\path{./tuning/}) and copy the example scenario files
located in the \code{examples} folder to the created directory:
%<<dir0,engine='bash',eval=FALSE>>=
\begin{lstlisting}[style=BashInputStyle]
mkdir ./tuning
cd ./tuning
# $IRACE_HOME is the installation directory of irace.
cp $IRACE_HOME/examples/acotsp/* ./
ls ./
# Make sure that target-runner is executable
chmod u+x target-runner
\end{lstlisting}
%@
\item Download the training instances from \url{https://iridia.ulb.ac.be/supp/IridiaSupp2016-003/scenarios/acotsp/instances.tar.gz} to the \path{./tuning/} directory and decompress it, which creates create a folder \path{instances}:
%<<instance0,engine='bash',eval=FALSE>>=
\begin{lstlisting}[style=BashInputStyle]
tar -xvf instances.tar.gz
ls instances/
\end{lstlisting}
%@
If the above gives an error or does not show any files, then the files were not extract correctly. Maybe the \path{instances.tar.gz} file did not download correctly or maybe it is not in the correct place. It should be within the folder \path{tuning}.correctly.
\item Download the \ACOTSP software from \url{https://github.com/MLopez-Ibanez/ACOTSPQAP/archive/refs/heads/master.zip} to the \path{./tuning/}
directory and compile the \path{acotsp} executable using \code{make}.
%<<acotsp0,engine='bash',eval=FALSE>>=
\begin{lstlisting}[style=BashInputStyle]
unzip master.zip
make -C ACOTSPQAP-master acotsp
./ACOTSPQAP-master/acotsp --version
\end{lstlisting}
%@
If the above gives an error, then the \path{acotsp} executable failed to compile for some reason. Maybe you are missing the C compiler or some files did not extract correctly.
\item Create a directory for executing the experiments and execute \irace:
%<<runexample,engine='bash',eval=FALSE>>=
\begin{lstlisting}[style=BashInputStyle]
mkdir ./acotsp-arena/
# $IRACE_HOME is the installation directory of irace.
$IRACE_HOME/bin/irace
\end{lstlisting}
%@
Or you can also execute \irace from the \aR console using:
<<runexample2,prompt=FALSE,eval=FALSE>>=
library("irace")
setwd("./tuning/")
irace_cmdline()
@
\end{enumerate}
The most usual sources of error when running the above commands are:
\begin{itemize}
\item The \irace package is not correctly installed. Please make sure that installing \irace did not give any errors.
\item The location of the files is not correct. Please make sure that you have:
\begin{itemize}
\item The folder \path{tuning} and that it contains the files \path{scenario.txt}, \path{parameters-acotsp.txt}, \path{target-runner}, and the folders \path{instances}, \path{ACOTSPQAP-master} and \path{acotsp-arena}.
\item The folder \path{instances} should contain the TSP instance files (\path{*.tsp}).
\item The folder \path{ACOTSPQAP-master} should contain the executable \path{acotsp}. You should be able to invoke \code{./ACOTSPQAP-master/acotsp --version} without an error.
\item The folder \path{acotsp-arena} should be empty.
\end{itemize}
\end{itemize}
%%
%%
%%
%% Scenario settings
%%
%%
%%
\section{Defining a configuration scenario}\label{sec:scenario}
\subsection{Target algorithm parameters} \label{sec:target parameters}
The parameters of the target algorithm are defined by a parameter file as
described in \autoref{sec:parameters file}. Optionally, when executing
\irace from the \aR console, the parameters can be specified directly as an \aR
object (see \autoref{sec:parameters object}). For defining your parameters
follow the guidelines provided in the following sections.
\subsubsection{Parameter types}
Each target parameter has an associated type that defines its domain and the
way \irace handles them internally. Understanding the nature of the domains of
the target parameters is important to select appropriate types. The four basic
types supported by \irace are the following:
\begin{itemize}
\item \textit{Real} parameters are numerical parameters that can take
floating-point values within a given range. The range is
specified as an interval `\code{(<lower bound>,<upper
bound>)}'. This interval is closed, that is, the parameter value
may eventually be one of the bounds. The possible values are rounded
to a number of \emph{decimal places} specified by
the global option \code{digits} (Section~\ref{sec:globaloptions}). For example, given the default
number of digits of $4$, the values $0.12345$ and
$0.12341$ are both rounded to $0.1234$.
Selected real-valued parameters can be optionally sampled on
a logarithmic scale (base $e$).
\item \textit{Integer} parameters are numerical parameters that can take only
integer values within the given range. Their range is specified as the range
of real parameters and they can also be optionally sampled on a
logarithmic scale (base $e$).
\item \textit{Categorical} parameters are defined by a set of possible values
specified as `\code{(<value 1>,} \code{...,} \code{<value n>)}'. The values
are quoted or unquoted character strings. Empty strings and strings
containing commas or spaces must be quoted.
\item \emph{Ordinal} parameters are defined by an \emph{ordered} set
of possible values in the same format as for categorical
parameters. They are handled internally as integer parameters, where
the integers correspond to the indexes of the values.
\end{itemize}
\begin{xwarningbox}
Boolean (or logical) parameters are best encoded as categorical ones with just two values rather than integer ones with domain $(0,1)$. Some boolean parameters take an explicit value (0/1 or true/false) such as:
\begin{CodeInput}
dlb "--dlb " c (0, 1)
\end{CodeInput}
Others are switches whose presence activates the parameter:
\begin{CodeInput}
dlb "" c ("", "--dlb")
\end{CodeInput}
\end{xwarningbox}
\subsubsection{Parameter domains}
For each target parameter, an interval or a set of values must be defined
according to its type, as described above. There is no limit for the size of
the set or the length of the interval, but keep in mind that larger domains
could increase the difficulty of the tuning task. Choose always values that you
consider relevant for the tuning. In case of doubt, we recommend to choose larger
intervals, as occasionally best parameter settings may be not intuitive a priori.
All intervals are considered as closed intervals.
It is possible to define parameters that will have always the same value.
Such ``\emph{fixed}'' parameters will not be tuned but their values are used
when executing the target algorithm and they are affected by constraints
defined on them. All fixed parameters must be defined as categorical
parameters and have a domain of one element.
\subsubsection{Parameter dependent domains}\label{sec:paramdependant}
Domains that are dependent on the values of other parameters can be specified
only for numerical parameters (both integer and real). To do so, the dependent
domain must be expressed in function of another parameter, which must be a
numerical parameter. The expression that defines a dependency must be written
between quotes: \code{(value, "expression")} or \code{("expression", value)} or
\code{("expression", "expression")}.
The expressions can only use the following operators and \aR functions: \code{+},
\code{-}, \code{*}, \code{/}, \code{\%\%}, \code{min}, \code{max},
\code{round}, \code{floor}, \code{ceiling}, \code{trunc}. If you need to use an
operator or function not listed here, please contact us.
\begin{xwarningbox}
The user must ensure that the defined domain is valid at all times since
\irace currently is not able to detect possible invalid domains based on the expressions
provided.
\end{xwarningbox}
If you have a parameter \code{p2} that is just a transformation of another
\code{p1}, then instead of using a dependent domain (left-hand side of the
following example), it will be better to create a dummy parameter that controls
the transformation (right-hand side) and do the transformation within \code{target-runner}. For example:
%
\begin{center}
\begin{minipage}{0.4\linewidth}
\small\centering%
\begin{CodeInput}[frame=single]
# With dependent domains
p1 "" r (0, 100)
p2 "" r ("p1", "p1 + 10")
\end{CodeInput}
\end{minipage}
\hspace{\stretch{1}} should be \hspace{\stretch{1}}
\begin{minipage}{0.4\linewidth}
\small\centering%
\begin{CodeInput}[frame=single]
# With a dummy parameter
p1 "" r (0, 100)
p2dum "" r (0, 10)
\end{CodeInput}
\end{minipage}
\end{center}
%
and \code{target-runner} will compute $\code{p2} = \code{p2dum} \cdot \code{p1}$.
\subsubsection{Conditional parameters}\label{sec:conditional}
Conditional parameters are active only when others have certain values. These
dependencies define a hierarchical relation between parameters. For example,
the target algorithm may have a parameter \code{localsearch} that takes values
\code{(sa, ts)} and another parameter \code{ts-length} that only needs to be
set if the first parameter takes precisely the value \code{ts}. Thus, parameter
\code{ts-length} is conditional on \code{localsearch == "ts"}.
\subsubsection{Forbidden parameter configurations}\label{sec:forbidden}
A line containing just \texttt{[forbidden]} ends the list of parameters and starts the list of forbidden expressions.
Each line is a logical expression (in \aR syntax) containing
parameter names as defined by the \parameter{parameterFile}
(\autoref{sec:target parameters}), values and logical operators. For a list of \aR logical operators see:
\begin{center}
\url{https://stat.ethz.ch/R-manual/R-devel/library/base/html/Syntax.html}
\end{center}
If \irace generates a parameter configuration that makes any of the logical
expressions evaluate to \code{TRUE}, then the configuration is considered
forbidden and it is never evaluated. This is useful when some combination
of parameter values could cause the target algorithm to crash, consume
excessive CPU time or memory, or when it is known that they do no produce
satisfactory results.
\begin{xwarningbox}
Initial configuration (\autoref{sec:initial}) that are forbidden will be discarded with a warning.
\end{xwarningbox}
If the forbidden constraints provided are too strict, \irace may produce the following error:
\begin{CodeInput}
irace tried 100 times to sample from the model a configuration not forbidden
without success, perhaps your constraints are too strict?
\end{CodeInput}
%
In that case, it may be a good idea to reformulate the forbidden constraints as conditional parameters (\autoref{sec:conditional}), parameter-dependent domains (\autoref{sec:paramdependant}), repairing the configurations (\autoref{sec:repairconf}) or post-processing within the target-algorithm (\autoref{sec:complex_domains}).
\subsubsection{Global options}\label{sec:globaloptions}
A line containing just \texttt{[global]} starts the definition of global options. The only global option currently implemented is \code{digits}, which controls the number of decimal digits for real valued parameters. Its default value is 4.
\subsubsection{Parameter file format}\label{sec:parameters file}
For simplicity, the description of the parameters space is given as a
table. Each line of the table defines a configurable parameter
%
\begin{center}
\code{<name>\ <label>\ <type>\ <domain>\ [ | <condition>\ ] }
\end{center}
where each field is defined as follows:
%
\begin{description}[left=5em,itemindent=-0.5em,align=right,itemsep=0pt]
% \DrawEnumitemLabel
\item[\code{<name>}] The name of the parameter as an unquoted
alphanumeric string, e.g., `\code{ants}'.
%
\item[\code{<label>}]
A \emph{label} for this parameter. This is a
string that will be passed together with the parameter
to \parameter{targetRunner}. In the default \parameter{targetRunner}
provided with the package (\autoref{sec:runner}), this is the
command-line switch used to pass the value of this parameter, for
instance `\code{"--ants "}'.\newline
The value of the parameter is concatenated \emph{without
separator} to the label when
invoking \parameter{targetRunner}, thus \emph{any whitespace in the label is significant}. Following the same example, when parameter \code{ants} takes value \code{5}, the default targetRunner will pass the parameter as \code{"--ants 5"}.
\item[\code{{<type>}}] The type of the parameter, either
\textit{integer}, \textit{real}, \textit{ordinal} or
\textit{categorical}, given as a single letter: `\code{i}',
`\code{r}', `\code{o}' or `\code{c}'. Numerical parameters
can be sampled using a natural logarithmic scale with '\code{i,log}' and
'\code{r,log}' (without spaces) for integer and real
parameters, respectively.
%
\item[\code{{<domain>}}] The range or set of values of the parameter delimited by
parentheses, \eg~\code{(0,1)} or \code{(a,b,c,d)}.
See also parameter dependent domains (\autoref{sec:paramdependant}).
%
\item[\code{{<condition>}}] An optional \emph{condition} that determines whether the parameter is
enabled or disabled, thus making the parameter conditional. If the
condition evaluates to false, then no value is assigned to this
parameter, and neither the parameter value nor the corresponding
label are passed to \parameter{targetRunner}. The condition must follow the same syntax as those for specifying forbidden configurations (see below), that is, it must be a
valid \aR logical expression\footnote{For a list of \aR operators
see: \url{https://stat.ethz.ch/R-manual/R-devel/library/base/html/Syntax.html}}.
The condition may contain the name
of other parameters as long as the dependency graph does not
contain any cycle. Otherwise, \irace will detect the cycle and
stop with an error.
\end{description}
\begin{xwarningbox}
Categorical and ordinal parameters are always treated as strings. Given a parameter like:
\begin{CodeInput}
a "" c (0, 5, 10, 20)
\end{CodeInput}
then, a condition like \code{a > 10} will be true when \code{a} is \code{5},
because comparisons between strings are lexicographic and \code{"10"} is sorted
before \code{"5"}. As a work-around, you can convert the string to numeric in
the condition with \code{as.numeric(a)}.
\end{xwarningbox}
As an example, Figure~\ref{fig:acotsp_parameters} shows the parameters file
of the \ACOTSP scenario.
\begin{figure}[!hbt]
\small\centering%
\begin{CodeInput}
# name switch type values [conditions (using R syntax)]
algorithm "--" c (as,mmas,eas,ras,acs)
localsearch "--localsearch " c (0, 1, 2, 3)
alpha "--alpha " r (0.00, 5.00)
beta "--beta " r (0.00, 10.00)
rho "--rho " r (0.01, 1.00)
ants "--ants " i (5, 100)
nnls "--nnls " i (5, 50) | localsearch %in% c(1, 2, 3)
q0 "--q0 " r (0.0, 1.0) | algorithm == "acs"
dlb "--dlb " c (0, 1) | localsearch %in% c(1,2,3)
rasrank "--rasranks " i (1, "ants") | algorithm == "ras"
elitistants "--elitistants " i (1, 750) | algorithm == "eas"
[forbidden]
## Examples of valid logical operators are:
## == != >= <= > < & | ! %in%
(alpha == 0.0) & (beta == 0.0)
\end{CodeInput}
\caption{Parameter file (\code{parameters.txt}) for tuning \ACOTSP.}\label{fig:acotsp_parameters}
\end{figure}
\subsubsection{Parameters R format}\label{sec:parameters object}
The target parameters are stored in an \aR~\code{list} that you can obtain from the \aR console using the following command:
<<readParameters,prompt=FALSE, eval=FALSE>>=
parameters <- readParameters(file = "parameters.txt")
@
See the help of the \code{readParameters} function (\code{?readParameters}) for more information.
The structure of the parameter list that is created is as follows:
\begin{center}
\renewcommand{\arraystretch}{1.2}
\begin{tabularx}{0.98\linewidth}{@{}rX}
\code{names} & Vector that contains the names of the parameters. \\
\code{types} & Vector that contains the type of each parameter 'i', 'c', 'r', 'o'. \\
\code{switches} & Vector that contains the labels of the parameters. e.g., switches to be used for the
parameters on the command line. \\
\code{domain} & List of vectors, where each vector may contain two
values (minimum, maximum) for real and integer parameters, or
a set of values for categorical and ordinal parameters.\\
\code{conditions} & List of \aR logical expressions, with variables
corresponding to parameter names.\\
\code{isFixed} & Logical vector that specifies which parameter is fixed
and, thus, it does not need to be tuned.\\
\code{transform} & Vector that contains the transformation of each parameter. Currently, it
can take values \code{``''} (no transformation, default) of \code{``log''}
(natural logarithmic transformation).\\
\code{nbParameters} & An integer, the total number of parameters.\\
\code{nbFixed} & An integer, the number of parameters with a fixed value.\\
\code{nbVariable} & Number of variable (i.e., to be tuned) parameters.\\
\code{isDependent} & Logical vector that specifies which parameter defines
a dependent domain.\\
\code{forbidden} & List of R logical expressions that cannot evaluate to \code{TRUE} for any evaluated configuration.\\
\end{tabularx}
\end{center}
The following example shows the structure of the \code{parameters} \aR object
for the \code{algorithm}, \code{ants} and \code{q0} parameters of the \ACOTSP scenario:
<<parametersetup,eval=TRUE,include=FALSE>>=
# Setup example
parameters <- readParameters(text='
algorithm "--" c (as,mmas,eas,ras,acs)
ants "--ants " i (5, 100)
q0 "--q0 " r (0.0, 1.0) | algorithm %in% c("acs")
')
@
<<parameterlist,eval=TRUE, prompt=TRUE, size='normalsize', comment="">>=
str(parameters, vec.len = 10)
@
\subsection{Target algorithm runner}
\label{sec:runner}
The evaluation of a candidate configuration on a single instance is done by means of a user-given auxiliary
program or, alternatively, a user-given \aR function. The function (or program name) is specified by the option
\parameter{targetRunner}. The \parameter{targetRunner} must return the
cost value (\eg cost of the best solution found) of the evaluation; unless computing the cost requires information from all the configurations evaluated on an instance, \eg when evaluating multi-objective algorithms with unknown normalisation bounds (see \autoref{sec:evaluator} for details).
\begin{xwarningbox}
The objective of \irace is to minimize the cost value returned by the target
algorithm. If you wish to maximize, you can multiply the cost by
\code{-1} before returning it to \irace.
\end{xwarningbox}
\subsubsection{Target runner as an executable program}
When \parameter{targetRunner} is an auxiliary executable program, it
is invoked for each candidate configuration, passing as arguments:
%
\begin{center}
\small\code{<id_configuration>\ <id_instance>\ <seed>\ <instance>\ [bound]\ <configuration>}
\end{center}
%
\begin{center}
\renewcommand{\arraystretch}{1.2}
\begin{tabularx}{1\linewidth}{@{}rX}
\code{id_configuration} & an alphanumeric string that uniquely identifies a configuration;\\
\code{id_instance} & an alphanumeric string that uniquely identifies an instance;\\
\code{seed} & seed for the random number generator to be used for this evaluation, ignore the seed for deterministic algorithms;\\
\code{instance} & string giving the instance to be used for this evaluation;\\
\code{bound} & optional execution time bound. Only provided when the \parameter{boundMax} option is set in the scenario, see \autoref{sec:capping};\\
\code{configuration} & the pairs parameter label-value that
describe this candidate configuration. Typically given as command-line
switches to be passed to the executable program.\\
\end{tabularx}
\end{center}
The experiment list shown in \autoref{sec:runner fx}, would result in the following execution line:
%<<target_runner_ex1, engine='bash', eval=FALSE>>=
\begin{lstlisting}[style=BashInputStyle]
target-runner 1 113 734718556 /home/user/instances/tsp/2000-533.tsp \
--eas --localsearch 0 --alpha 2.92 --beta 3.06 --rho 0.6 --ants 80
\end{lstlisting}
%@
The command line switches that describe the candidate configuration are
constructed by appending to each parameter label (switch), \emph{without
separator}, the value of the parameter, following the order given in the
parameter table. The program \parameter{targetRunner} must print a real
number, which corresponds to the cost measure of the candidate configuration
for the given instance and optionally its execution time (mandatory when \parameter{maxTime}
is used and/or when the \parameter{capping} option is enabled). The working directory of
\parameter{targetRunner} is set to the execution directory specified by the option
\parameter{execDir}. This allows the user to execute independent runs of \irace
in parallel using different values for \parameter{execDir}, without the runs
interfering with each other.
\subsubsection{Target runner as an R function}
\label{sec:runner fx}
When \parameter{targetRunner} is an \aR function, it is invoked
for each candidate configuration as:
%
<<targetRunner,prompt=FALSE, eval=FALSE>>=
targetRunner(experiment, scenario)
@
%
\noindent where \code{experiment} is a list that contains information about
configuration and instance to execute one experiment, and \code{scenario} is the scenario
list. The structure of the \code{experiment} list is as follows:
\begin{center}
\renewcommand{\arraystretch}{1.2}
\begin{tabularx}{1\linewidth}{@{}rX}
\code{id_configuration} & an alphanumeric string that uniquely identifies a configuration;\\
\code{id_instance} & an alphanumeric string that uniquely identifies an instance;\\
\code{seed} & seed to be used for this evaluation;\\
\code{instance} & string giving the instance to be used for this evaluation;\\
\code{bound} & optional execution time bound;\\
\code{configuration} & 1-row data frame with a column per parameter name;\\
\end{tabularx}
\end{center}
The following is an example of an experiment list for the \ACOTSP scenario:
<<experimentlist,eval=TRUE,size='normalsize', prompt=TRUE, comment="">>=
print(experiment)
@
If you need the \code{switches} of each parameter (or any other information
about the parameters), you can obtain it from \code{scenario$parameters}.
You can find an example that calls MATLAB from \aR using this approach here: \url{https://github.com/MLopez-Ibanez/irace/blob/master/inst/examples/matlab/scenario.txt}.
If \parameter{targetEvaluator} is \code{NULL}, then
the \parameter{targetRunner} function must return a list with at least one
element \code{"cost"}, the numerical value corresponding to the evaluation of
the given configuration on the given instance. A cost of \code{Inf} is accepted
and results in the immediate rejection of the configuration (see
\autoref{sec:reject}).
If the scenario option \parameter{maxTime} is non-zero or if the \parameter{capping}
option is enabled, then the list must contain at least another element \code{"time"} that
reports the execution time for this call to \code{targetRunner}.
The return list may also contain the following optional elements that are used
by \irace for reporting errors in \code{targetRunner}:
%
\begin{center}
\renewcommand{\arraystretch}{1.2}
\begin{tabularx}{1\linewidth}{@{}rX}
\code{error} & is a string used to report an error;\\
\code{outputRaw} & is a string used to report the raw output of calls to
an external program or function;\\
\code{call} & is a string used to report how \code{targetRunner} called
an external program or function;\\
\end{tabularx}
\end{center}
\subsection{Target evaluator} \label{sec:evaluator}
Normally, \parameter{targetRunner} returns the cost of the execution of a
candidate configuration (see \autoref{sec:runner}). However, there are cases
when the cost evaluation must be delayed until all candidate configurations in
a race have been executed on a instance.
The \parameter{targetEvaluator} option defines an auxiliary program (or an \aR
function) that allows postponing the evaluations of the candidate
configurations. For each instance seen, the program \parameter{targetEvaluator} is only invoked after all
the calls to \parameter{targetRunner} for all alive candidate configurations on
the same instance have already finished.
\begin{xwarningbox}
When using \parameter{targetEvaluator}, \parameter{targetRunner} must not return the evaluation
of the configuration. If \parameter{maxTime} is used, \parameter{targetRunner} must return only
execution time.
\end{xwarningbox}
As an example, \parameter{targetEvaluator} may be used to dynamically find
normalization bounds for the output returned by an algorithm for each
individual instance. In this case, \parameter{targetRunner} will save the
output of the algorithm, then the first call to \parameter{targetEvaluator}
will examine the output produced by all calls to \parameter{targetRunner} for
the same instance, update the normalization bounds and return the normalized
output. Subsequent calls to \parameter{targetEvaluator} for the same instance
will simply return the normalized output.
A similar need arises when using quality measures for multi-objective
optimization algorithms, such as the hypervolume, which typically
require specifying reference points or sets. By
using \parameter{targetEvaluator}, it is possible to dynamically compute the
reference points or sets while \irace is running. Examples are provided at
\path{examples/hypervolume}. See also \autoref{sec:multi objective} for
more information on how to tune multi-objective algorithms.
\subsubsection{Target evaluator executable program}
When \parameter{targetEvaluator} is an auxiliary executable program, it
is invoked for each candidate with the following arguments:
%
\begin{center}
\small\code{<id_configuration>\ <id_instance>\ <seed>\ <instance>\ <num_configurations>\ <all_conf_id>}
\end{center}
%
\begin{center}
\renewcommand{\arraystretch}{1.2}
\begin{tabularx}{1\linewidth}{@{}rX}
\code{id_configuration} & an alphanumeric string that uniquely identifies a configuration;\\
\code{id_instance} & an alphanumeric string that uniquely identifies an instance;\\
\code{seed} & seed to be used for this evaluation;\\
\code{instance} & string giving the instance to be used for this evaluation;\\
\code{num_configurations} & number of alive candidate configurations;\\
\code{all_conf_id} & list of IDs of the alive configurations separated by whitespace.\\
\end{tabularx}
\end{center}
The \parameter{targetEvaluator} executable must print a numerical value
corresponding to the cost measure of the candidate configuration on
the given instance.
\subsubsection{Target evaluator R function}
When \parameter{targetEvaluator} is an \aR function, it is invoked
for each candidate configuration as:%
%
<<targetEvaluator, eval=FALSE>>=
targetEvaluator(experiment, num_configurations, all_conf_id, scenario,
target_runner_call)
@
%
\noindent where \code{experiment} is a list that contains information about one
experiment (see \autoref{sec:runner fx}), \code{num_configurations}
is the number of configurations alive in the race, \code{all_conf_id} is
the vector of IDs of the alive configurations, \code{scenario} is
the scenario list and \code{target_runner_call} is the string of the
\parameter{targetRunner} execution line.
The function \parameter{targetEvaluator} must return a list with one element
\code{"cost"}, the numerical value corresponding to the cost measure of the
given configuration on the given instance.
The return list may also contain the following optional elements that are used
by \irace for reporting errors in \code{targetEvaluator}:
%
\begin{center}
\renewcommand{\arraystretch}{1.2}
\begin{tabularx}{1\linewidth}{@{}rX}
\code{error} & is a string used to report an error;\\
\code{outputRaw} & is a string used to report the raw output of calls to
an external program or function;\\
\code{call} & is a string used to report how \code{targetEvaluator} called
an external program or function;\\
\end{tabularx}
\end{center}
\subsection{Training instances}\label{sec:training}
The \irace options \parameter{trainInstancesDir} and \parameter{trainInstancesFile}
specify where to find the training instances.
%
If you only set the value of \parameter{trainInstancesDir} (e.g., to \path{./Instances}), \irace will consider all files within that directory as training instances.
Otherwise, the value of \parameter{trainInstancesFile} may specify a text
file. The format of this file is one instance per line. Within each line, elements separated by white-space will be parsed as separate arguments to be supplied to \parameter{targetRunner}. This allows defining instance-specific parameter settings. Quoted strings will be parsed as a single argument.
%
The following example shows a training instance file for the \ACOTSP scenario:
\begin{figure}[!ht]
\centering%
\begin{minipage}{0.4\textwidth}
\begin{CodeInput}
# Example training instances file
100/100-1_100-2.tsp --time 1
100/100-1_100-3.tsp --time 2
100/100-1_100-4.tsp --time 3
\end{CodeInput}
\end{minipage}
\caption{Training instances file for tuning \ACOTSP.}\label{fig:acotsp_training}
\end{figure}
The value of \parameter{trainInstancesDir}, if set, is always prefixed to the instance
name, that is, the instances names are treated as relative to this directory.
%
For example, given the above file as \parameter{trainInstancesFile} and \parameter{trainInstancesDir}\code{="./Instances"}, then a possible invocation of \parameter{targetRunner} would be:
%
%<<target_runner_cmd, engine='bash', eval=FALSE>>=
\begin{lstlisting}[style=BashInputStyle]
target-runner 1 4 5718 ./Instances/100/100-1_100-2.tsp --time 1 --alpha 2.92 ...
\end{lstlisting}
%@
Training instances do not need to be files, \irace just passes the elements of each line as arguments to \parameter{targetRunner}, thus each line may denote the name of a benchmark function or a label, plus instance-specific settings, that the target algorithm understands. Each line may even be the command-line parameters required to call an instance generator within \parameter{targetRunner}. When the instances do not represent actual files, then \parameter{trainInstancesDir} is usually set to the empty string (\code{--train-instances-dir=""}). For example,
\begin{center}
\begin{minipage}{0.4\linewidth}
\begin{CodeInput}
# Example training instances file
rosenbrock_20 --function=12 --nvar 20
rosenbrock_30 --function=12 --nvar 30
rastrigin_20 --function=15 --nvar 20
rastrigin_30 --function=15 --nvar 30
\end{CodeInput}
\end{minipage}
\end{center}
%\caption{Example of training instances file where instances do not correspond to files.}\label{fig:nofile_training}
%\end{figure}
Optionally, when executing \irace from the \aR console, the list of instances might be provided explicitly by means of the variable \code{scenario$instances}. Thus, the previous example would be equivalent to:
<<instance1, prompt=FALSE, eval=FALSE>>=
scenario$instances <- c("rosenbrock_20 --function=12 --nvar 20",
"rosenbrock_40 --function=12 --nvar 30",
"rastrigin_20 --function=15 --nvar 20",
"rastrigin_40 --function=15 --nvar 30")
@
By default, \irace assumes that the target algorithm is stochastic (the value
of the option \parameter{deterministic} is $0$), thus, the same configuration
can be executed more than once on the same instance and obtain different
results. In this case, \irace generates pairs \code{(instance, seed)} by
generating a random seed for each instance. In other words, configurations
evaluated on the same instance use the same random seed. This is a well-known
variance reduction technique called \emph{common random
numbers}~\citep{McG1992vrt}. If all available pairs are used within a run of
\irace, new pairs are generated with different seeds, that is, a configuration
evaluated more than once per instance will use different random seeds.
If \parameter{deterministic} is set to $1$, then each instance will be used at
most once per race. This setting should only be used for target algorithms that
do not have a stochastic behavior and, therefore, executing the target
algorithm on the same instance several times with different seeds does not make
sense.
\begin{xwarningbox}
If \parameter{deterministic} is active and the number of training instances
provided to \irace is less than \parameter{firstTest} (default: 5), no
statistical test will be performed on the race.
\end{xwarningbox}
Finally, \irace randomly re-orders the sequence of instances provided. This
random sampling may be disabled by using the option \parameter{sampleInstances}
(\code{--sample-instances 0}) if keeping the order provided in the instance
file is important.
\begin{xwarningbox}
We advise to always sample instances to prevent biasing the tuning due to the
instance order. See also \autoref{sec:het}
\end{xwarningbox}
\subsection{Initial configurations} \label{sec:initial}
The scenario option \parameter{configurationsFile} allows specifying a text
file that contains an initial set of configurations to start the execution of
\irace. If the number of initial configurations supplied in the file is less
than the number of configurations required by \irace in the first iteration,
additional configurations will be sampled uniformly at random.
The format of the configurations file is one configuration per line, and one
parameter value per column. The first line must give the parameter name
corresponding to each column (names must match those given in the parameters
file). Each configuration must satisfy the parameter conditions (\code{NA}
should be used for those parameters that are not enabled for a given
configuration) and not be forbidden by the constraints that define forbidden
configurations (\autoref{sec:forbidden}), if any.
Figure~\ref{fig:acotsp_default} gives an example file that corresponds to the
\ACOTSP scenario.
\begin{figure}[!ht]
\centering
\begin{minipage}{0.7\linewidth}
\footnotesize
\begin{CodeInput}
## Initial candidate configuration for irace
algorithm localsearch alpha beta rho ants nnls dlb q0 rasrank elitistants
as 0 1.0 1.0 0.95 10 NA NA 0 NA NA
\end{CodeInput}
\end{minipage}
\caption{Initial configuration file (\code{default.txt}) for tuning \ACOTSP.}\label{fig:acotsp_default}
\end{figure}
We advise to use this feature when a default configuration of the target algorithm exists or when
different sets of good parameter values are known. This will allow \irace to start the search
from those parameter values and attempt to improve their performance.
\subsection{Repairing configurations}\label{sec:repairconf}
In some problems, the parameter values require complex constraints that cannot
be implemented by constraints defined in the parameter space
(\autoref{sec:forbidden}).
%
The scenario option \parameter{repairConfiguration} can be set to a
user-defined \aR function that takes a single configuration generated by irace
and returns a ``\emph{repaired}'' configuration, thus allowing the
implementation of any rules necessary to satisfy arbitrary constraints on
parameter values.
%
The \parameter{repairConfiguration} function is called after generating a
configuration and before checking for forbidden configurations. The first
argument is a 1-row \code{data.frame} with parameter names as the column names and the second argument is the \code{parameters} list (\autoref{sec:parameters
object}). An example that makes all real-valued parameters
sum up to one would be:
%
<<repairEx,prompt=FALSE, eval=FALSE>>=
repairConfiguration = function(configuration, parameters)
{
isreal <- names(which(parameters$types[colnames(configuration)] == "r"))
# This ignores 'digits'
c_real <- unlist(configuration[isreal])
c_real <- c_real / sum(c_real)
configuration[isreal] <- c_real
return(configuration)
}
@
The following example forces three specific parameters to be in increasing order:
%
<<repairEx2,prompt=FALSE, eval=FALSE>>=
repairConfiguration = function(configuration, parameters)
{
columns <- c("p1","p2","p3")
# cat("Before"); print(configuration)
configuration[columns] <- sort(unlist(configuration[columns], use.names=FALSE))
# cat("After"); print(configuration)
return(configuration)
}
@
The above code can be specified directly in the \parameter{scenarioFile}, by default \code{scenario.txt}.
%%
%%
%%
%% Parallel processing
%%
%%
%%
\section{Parallelization}\label{sec:parallel}
A single run of \irace can be done much faster by executing the calls
to \parameter{targetRunner} (the runs of the target algorithm) in
parallel. There are four ways to parallelize a single run of \irace:
\begin{enumerate}
\item \textbf{Parallel processes}: The option \parameter{parallel} executes
multiple calls to \parameter{targetRunner} in parallel within a single
computer, by means of the \pkg{parallel} \aR package. For example, adding
\code{--parallel N} to the command line of \irace will launch in parallel up
to $N$ calls of the target algorithm. When using this option within a
computing cluster, \irace will be submitted as a \emph{job} in some way that tells the cluster to ``reserve'' $N$ CPUs (or tasks depending on the cluster) within a single cluster node (a single machine).
\item \textbf{MPI}: By enabling the option \parameter{mpi}, calls
to \parameter{targetRunner} will be executed in parallel by using the message
passing interface (MPI) protocol (requires the \pkg{Rmpi} \aR package). In
this case, the option \parameter{parallel} controls the number of slave nodes
used by \irace. For example, adding \code{--mpi 1 --parallel N} to the
command-line will create $N$ slaves + 1 master, and execute up to $N$ calls of
\parameter{targetRunner} in parallel.
The user is responsible for setting up the required MPI environment. MPI is
commonly available in computing clusters and requires launching \irace in
some particular way. An example script for using MPI mode in a SGE cluster
is given at \IRACEHOME{/bin/parallel-irace-mpi}.
By default, \irace dynamically balances the load among nodes, however, this
may significantly increase communication overhead in some parallel
environments and disabling \parameter{loadBalancing} may be faster.
\item \textbf{Batch queue mode}: Some computing clusters work by submitting
jobs to a batch queue and waiting for the jobs to finish. With the
option \parameter{batchmode} (\code{--batchmode} \code{[sge|pbs|} \code{torque|slurm]}),
\irace will launch in parallel as many calls of \parameter{targetRunner} as
possible and use a
cluster-specific method to wait for jobs to finish. In this mode, \parameter{parallel} controls how many jobs are queued and should be set to the queue limit of your cluster. If your cluster type is
not supported or not working as expected, please submit a pull request (\url{https://github.com/MLopez-Ibanez/irace/pulls}) adding support to your cluster type.
See the examples in
\IRACEHOME{/examples/batchmode-cluster/}.
\begin{xwarningbox}
In batchmode, \irace runs in the submission node of the cluster,
hence, \irace is not submitted to the cluster as a job (that is,
neither \code{qsub} nor \code{squeue} should be used to invoke \irace
itself). The user must call the appropriate job submission command (e.g., \code{qsub} or \code{squeue}) from \parameter{targetRunner} with the appropriate settings for their cluster, that is, \parameter{targetRunner} submits one job to the
cluster and prints a single string: The job ID that allows
\irace to determine the status of the running job. Moreover, the use of a
separate
\parameter{targetEvaluator} script is required to collect the results of
\parameter{targetRunner} and return them to \irace.
\end{xwarningbox}
\item \parameter{targetRunnerParallel}: This option allows users to fully
control the parallelization of the execution of \parameter{targetRunner}.
Its value must be an \aR function that will be invoked by \irace as follows:
<<targetRunnerParallel,prompt=FALSE, eval=FALSE>>=
targetRunnerParallel(experiments, exec_target_runner, scenario, target_runner)
@
%
where \code{scenario} is the list describing the configuration scenario
(\autoref{sec:scenario}); \code{experiments} is a list that describes the
configurations and instances to be executed (see \autoref{sec:runner} for a
description); \code{target_runner} is the function that calls the target
algorithm and it is the same as \parameter{targetRunner}, if the latter is a
function, or it is a call to \code{target_runner_default}, if
\parameter{targetRunner} is the path to an executable; and
\code{exec_target_runner} is an internal function within \irace that takes
care of executing \code{target_runner}, check its output and, possibly,
retry in case of error (see \parameter{targetRunnerRetries}). The
\parameter{targetRunnerParallel} function should call the given
\code{target_runner} function for each element in the \code{experiments}
list, possibly using \code{exec_target_runner} as a wrapper. A trivial
example would be:
<<targetRunnerParallel2,prompt=FALSE, eval=FALSE>>=
targetRunnerParallel <- function(experiments, exec_target_runner, scenario,
target_runner)
{
lapply(experiments, exec_target_runner, scenario = scenario,
target_runner = target_runner)
}
@
%
However, the user is free to set up the calls in any way, perhaps
implementing their own replacement for \code{target_runner} and/or
\code{exec_target_runner}. The user may load and call other \aR packages, such as \pkg{batchtools} (\url{https://mllg.github.io/batchtools/}).
The only requirement is that the \parameter{targetRunnerParallel} function
must return a list of the same length as \code{experiments}, where each
element is the output expected from the corresponding call
to \parameter{targetRunner} (see \autoref{sec:runner}).
The following is an example of the output of a call to
\parameter{targetRunnerParallel} with 2 experiments, in which the execution
time is not reported:
<<targetRunnerParallel3,prompt=FALSE, eval=TRUE>>=
print(output)
@
%
\end{enumerate}
The best option will depend on the resources available to you. Option 1 is
usually the fastest and simplest to setup. Running on a node (machine) with
128 CPUs will be faster than running on 8 nodes with 16 CPUs because the
communication between nodes required by MPI can be very slow depending on the
cluster. Option 2 may be faster if irace generates more configurations per
iteration than the number of CPUs of a single node. However, depending on the
configuration of your cluster, requesting many CPUs may require waiting in the
queue a long time. Option 3 may be the slowest since irace has to check the
queue frequently. However, irace will start running experiments as soon as 1
CPU is available, thus option 3 may actually finish earlier than the other
options if there is always some CPUs available in the cluster but the queue
for requesting many CPUs at once is very long.
As a rule-of-thumb, if you only have access to a single machine, then you only need option 1. If you have access to a computing cluster with multiple machines, then use option 1 with the maximum number of CPUs that a single node has in your computing cluster. If that number is 64 or more, it should be enough unless a single run of irace evaluates thousands of configurations. Otherwise, investigate option 2. If option 2 does not work, then investigate option 3.
%%
%%
%%
%% Testing
%%
%%
%%
\section{Testing (Validation) of configurations}\label{sec:testing}
Once the tuning process is finished, \irace returns a set of configurations
corresponding to the elite configurations at the end of the run, ordered from
best to worst. In order to evaluate the generality of these configurations
without looking at their performance on the training set, \irace offers the
possibility of evaluating these configurations on a test instance set,
typically different from the training set used during the tuning phase. These
evaluations will use the same settings for parallel
execution, \parameter{targetRunner} and \parameter{targetEvaluator}.
The test instances can be specified by the
options \parameter{testInstancesDir} and/or \parameter{testInstancesFile}, or by
setting directly the variable \code{scenario\$testInstances}. These options behave similarly to their counterparts for the training instances
(\autoref{sec:training}). In particular, each test instance is assigned a
different seed in the same way as done for the training instances. In
principle, \irace evaluates each configuration on each testing instance just
once, because evaluating one run on $n$ instances is always better than
evaluating $n'$ runs on $n/n'$ instances~\citep{IRIDIA-2004-001}. However, if
the number of instances is limited, one can always duplicate instances as
needed in the \parameter{testInstancesFile}, and \irace will assign a different
random seed to each instance.
%
An example of the output produced by \irace when testing is shown in Fig.~\ref{fig:testing_output}.
\afterpage{\clearpage
\begin{figure}[p]
\RecustomVerbatimCommand{\VerbatimInput}{VerbatimInput}%
{fontsize=\scriptsize,
%
frame=none, % top and bottom rule only
framesep=0pt, % separation between frame and text
rulecolor=\color{Gray},
}
% FIXME: This doesn't break long lines, find another alternative.
{\ttfamily\scriptsize\ldots
\VerbatimInput{irace-acotsp-testing.txt}}%
\caption{Sample text output of \irace when evaluating on test instances.}\label{fig:testing_output}
\end{figure}}
The options \parameter{testNbElites} and \parameter{testIterationElites}
control which configurations are evaluated during the testing phase. In
particular, setting \code{testIterationElites = 1} will test not only the final
set of elite configurations (those returned at the end of the training phase),
but also the set of elites at the end of each race
(iteration). The option \parameter{testNbElites} limits the maximum number of
configurations considered within each set. Some examples:
\begin{itemize}
\item \code{testIterationElites = 0; testNbElites = 1} means that only the best
configuration found during the run of \irace, the final best, will be used in
the testing phase.
\item \code{testIterationElites = 1; testNbElites = 1} will test, in addition to the final best, the best configuration found at each iteration.
\item \code{testIterationElites = 1; testNbElites = 2} will test the two best
configurations found at each iteration, in addition to the final best and
second-best configurations.
\end{itemize}
The testing can be also (re-)executed at a later time by using the following
\aR command (but you may need to override \code{testNbElites} and \code{testIterationElites}):
%
<<testing_r, prompt=FALSE, eval=FALSE>>=
testing_fromlog(logFile = "./irace.Rdata", testNbElites = 1)
@
%
The above line will load the scenario setup from \parameter{logFile} to
perform the testing. The testing results will be stored in the \aR object
\code{iraceResults\$testing}, which is saved in the file specified by
\code{scenario\$logFile}. The structure of the object is described in
\autoref{sec:output r}. For examples on how to analyse the results see
\autoref{sec:analysis}.
Another alternative is to test a specific set of configurations using the command-line option \parameter{--only-test} as follows:
\begin{lstlisting}[style=BashInputStyle]
irace --only-test configurations.txt
\end{lstlisting}
where \path{configurations.txt} has the same format as the
set of initial configurations (\autoref{sec:initial}).
%%
%%
%%
%% Recovery
%%
%%
%%
\section{Recovering \irace runs}\label{sec:recovery}
Problems like power cuts, hardware malfunction or the need to use computational
power for other tasks may occur during the execution of \irace, terminating a
run before completion. At the end of each iteration, \irace saves an \aR data
file (\parameter{logFile}, by default \code{"./irace.Rdata"}) that not only
contains information about the tuning progress (\autoref{sec:output r}), but
also internal information that allows recovering an incomplete execution.
To recover an incomplete \irace run, set the option \parameter{recoveryFile} to
the log file previously produced, and \irace will continue the execution from
the last saved iteration. The state of the random generator is saved and
loaded, therefore, as long as the execution is continued in the same machine,
the obtained results will be exactly the same as executing \irace in one step
(external factors, such as CPU load and disk caches, may affect the
target algorithm and that may affect the results). You can specify
the \parameter{recoveryFile} from the command-line or from the scenario file,
and execute \irace as described in \autoref{sec:execution}. For example, from
the command-line use:
\begin{lstlisting}[style=BashInputStyle]
irace --recovery-file "./irace-backup.Rdata"
\end{lstlisting}
\begin{xwarningbox}
When recovering a previous run, \irace will try to save data on the file
specified by the \parameter{logFile} option. Thus, you must specify different
files for \parameter{logFile} and \parameter{recoveryFile}. Before
recovering, we strongly advise to rename the saved \aR data file as in the
example above, which uses \code{"irace-backup.Rdata"}.
\end{xwarningbox}
\hide{%
%\LESLIE{Actually, there other variables we allow to change...what we shoudl say about this?}
% MANUEL: I would say nothing for now until we fix this properly.
When recovery is done you can modify some \irace options from the command-line
or from the scenario file.
\begin{itemize}
\item \parameter{execDir}
\item \parameter{logFile}
\item \parameter{debugLevel}
\item \parameter{parallel}
\item \parameter{loadBalancing}
\item \parameter{mpi}
\item \parameter{batchmode}
\end{itemize}
It is not possible (and in most cases not correct) to change the scenario options, given that the
previous results can become invalid. However there are some cases in which this options must be changed,
for example, if you have instances that are files, it might be the case that the instances are not
available at the same location. To change the location of them you should modify directly the \aR data file,
please be careful not to change the order of the instances, because this would make the results obtained by
\irace invalid. To change the instances path, open the \aR console and use:
<<change_recover, prompt=TRUE, eval=FALSE>>=
iraceResults <- read_logfile("./tuning/irace.Rdata")
new_path <- "./experiments/tuning/instances/"
iraceResults$scenario$instances <-
paste0(new_path, basename(iraceResults$scenario$instances))
save(iraceResults, file="./tuning/irace.Rdata")
@
This example can also can be applied to the target execution files \parameter{targetRunner} and \parameter{targetEvaluator}.
}
\begin{xwarningbox}
Do not change anything in the log file or the scenario file before
recovering, as it may have unexpected effects on the recovered run of
\irace. In case of doubt, please contact us first (\autoref{sec:contact}). In particular, it is not possible to continue a run of \irace by recovering with a larger budget. Results will \textbf{not} be the same as running \irace from the start with the largest budget. An alternative is to use the final configurations from one run as the initial configurations of a new run.
\end{xwarningbox}
\begin{xwarningbox}
If your scenario uses \parameter{targetEvaluator} (\autoref{sec:evaluator})
and \parameter{targetEvaluator} requires files created
by \parameter{targetRunner}, then recovery will fail if those files are not
present in the \parameter{execDir} directory. This can happen, for example,
if you recover from a different directory than the one from which irace was
initially executed, or when \parameter{execDir} is set to a temporary
directory for every irace run. Thus, you need to copy the contents of the
previous \parameter{execDir} into the new one.
\end{xwarningbox}
%%
%%
%%
%% Output and analysis of results
%%
%%
%%
\section{Output and results}
During its execution, \irace prints information about the progress of the
tuning in the standard output. Additionally, after each iteration, an \aR data
file is saved (\parameter{logFile} option) containing the state of \irace.
\subsection{Text output}\label{sec:output text}
Figure~\ref{fig:output} shows the output, up to the end of the first iteration,
of a run of elitist \irace applied to the \ACOTSP scenario with 1000
evaluations as budget.
\afterpage{\clearpage
\begin{figure}[p]
\RecustomVerbatimCommand{\VerbatimInput}{VerbatimInput}%
{fontsize=\scriptsize,
%
frame=none, % top and bottom rule only
framesep=0pt, % separation between frame and text
rulecolor=\color{Gray},
firstline=1,
lastline=68,
}
% FIXME: This doesn't break long lines, find another alternative.
\VerbatimInput{irace-acotsp-stdout.txt}%
\caption{Sample text output of \irace.}\label{fig:output}
\end{figure}}
First, \irace gives the user a warning informing that it has found a file with
the default scenario filename and it will use it. Then, general information
about the selected \irace options is printed:
\begin{itemize}
\item \parameter{nbIterations} indicates the minimum number of iterations \irace has calculated
for the scenario. Depending on the development of the tuning the final iterations that
are executed can be more.
\item \parameter{minNbSurvival} indicates the minimum number of alive configurations that
are required to continue a race. When less configurations are alive the race is
stopped and a new iteration begins.
\item \code{nbParameters} is the number of parameters of the scenario.
\item \parameter{seed} is the number that was used to initialize the random number generator in \irace.
\item \code{confidence level} is the confidence level of the statistical test.
\item \code{budget} is the total number of evaluations available for the tuning.
\item \code{time budget} is the maximum execution time available for the tuning.
\item \parameter{mu} is a value used for calculating the minimum number of iterations.
\item \parameter{deterministic} indicates if the target algorithm is assumed to be deterministic.
\end{itemize}
At each iteration, information about the progress of the execution is printed as follows:
\begin{itemize}
\item \code{experimentsUsedSoFar} is the number of experiments from the total budget
that have been used up to the current iteration.
\item \code{timeUsed} is the execution time used so far in the experiments. Only available when reported
in the \parameter{targetRunner} (activate it with the \parameter{maxTime} option).
\item \code{remainingBudget} is the number of experiments that have not been used yet.
\item \code{timeEstimate} estimation of the mean execution time. This is used to calculate
the remaining budget when \parameter{maxTime} is used.
\item \code{currentBudget} is the number of evaluations \irace has allocated to the
current iteration.
\item \parameter{nbConfigurations} is the number of configurations \irace will use in the
current iteration. In the first iteration, this number of configurations include
the initial configurations provided; in later iterations, it includes the elite
configurations from the previous iterations.
\end{itemize}
After the iteration information, a table shows the progress of the iteration execution.
Each row of the table gives information about the execution of an instance in the race.
The first column contains a symbol that describes the results of the
statistical test:
%
\begin{description}
\item \code{|x|} No statistical test was performed for this instance. The options \parameter{firstTest}
and \parameter{eachTest} control on which instances the statistical test is performed.
\item \code{|-|} Statistical test performed and configurations have been discarded. The column \code{Alive} gives an indication of how
many configurations have been discarded.
\item \code{|=|} Statistical test performed and no configurations have been discarded.
This means \irace needs to evaluate more instances to identify the best configurations.
\item \code{|!|} This indicator exists only for the elitist version of \irace. It indicates that
the statistical test was performed and some elite configurations appear to show bad performance and
could be discarded but they are kept because of the elitist rules. See option \parameter{elitist} in \autoref{sec:irace options} for more information.
\end{description}
Other columns have the following meaning:
\begin{description}
\item \code{Instance}: Index of \code{(instance, seed)} pair executed. This number
corresponds to the row in the data frame returned by \code{get_instanceID_seed_pairs()}. See \autoref{sec:output r}
for more information. This index is different from the instance ID passed to \parameter{targetRunner}.
\item \code{Bound}: Only when \parameter{capping} is enabled. Execution time used as bound
for the execution of new candidate configurations.
\item \code{Alive}: Number of configurations that have not been
discarded after the statistical test was performed.
\item \code{Best}: ID of the best configuration according to the instances seen
so far in this race (i.e., not including previous iterations).
\item \code{Mean best}: Mean cost value of the best configuration across the instances seen so
far in this race (not globally). Equivalent to the concept of ``iteration-best'' in other algorithms.
\item \code{Exp so far}: Number of experiments
performed so far.
\item \code{W time}: Wall-clock time spent on this instance.
\item \code{rho}, \code{KenW}, and \code{Qvar}: Spearman's rank correlation
coefficient rho, Kendall's concordance coefficient W, and a variance measure
described in \cite{SchHoo2012quanti}, respectively, of the configurations
across the instances evaluated so far in this iteration. These measures
evaluate how consistent is the performance of the configurations across the
instances. Values close to 1 for \code{rho} and \code{KenW} and values close
to 0 for \code{Qvar} indicate that the scenario is highly homogeneous. For
heterogeneous scenarios, we provide advice in \autoref{sec:het}.
\end{description}
Finally, \irace outputs the best configuration found and a list of the elite
configurations. The elite configurations are configurations that did not show
statistically significant difference during the race; they are ordered
according to their mean performance on the executed instances.
\subsection[R data file (logFile)]{\aR data file (\code{logFile})}\label{sec:output r}
The \aR data file created by \irace (by default as \code{irace.Rdata}, see option \parameter{logFile}) contains an object called
\code{iraceResults}. You can load this file in the \aR console with:
<<load_rdata, prompt=FALSE, eval=FALSE>>=
logfile <- system.file(package="irace", "exdata", "irace-acotsp.Rdata", mustWork=TRUE)
iraceResults <- read_logfile(logfile)
@
The \code{iraceResults} object is a list, and the elements of a list can be
accessed in \aR by using the \code{\$} or \code{[[]]} operators:
<<show_version, prompt=TRUE, eval=TRUE, comment="">>=
iraceResults$irace_version
@
\noindent%
The \code{iraceResults} list contains the following elements:
\begin{itemize}
\item \code{scenario}: The scenario \aR object containing the \irace options
used for the execution. See \autoref{sec:irace options} and the help
of the \pkg{irace} package; open an \aR console and type: \code{?defaultScenario}.
See \autoref{sec:irace options} for more information.
\item \code{parameters}: The parameters \aR object containing the description of
the target algorithm parameters. See \autoref{sec:target parameters}.
\item \code{allConfigurations}: The target algorithm configurations generated by
\irace. This object is a \code{data frame}, each row is a candidate
configuration; the first column (\code{.ID.}) indicates the internal identifier of the
configuration; the final column (\code{.PARENT.}) is the identifier
of the configuration from which the current configuration was sampled; and the remaining columns correspond to the parameter values; each column is named as
the parameter name specified in the parameter object.
<<show_configurations, prompt=TRUE, eval=TRUE, comment="">>=
head(iraceResults$allConfigurations)
@
\item \code{allElites}: A list that contains one element per iteration. Each element contains
the internal identifier of the elite candidate configurations of the corresponding iteration
(identifiers correspond to \code{allConfigurations$.ID.}).
<<show_idelites, prompt=TRUE, eval=TRUE, comment="">>=
print(iraceResults$allElites)
@
The configurations are ordered by mean performance, that is, the ID of the best
configuration corresponds to the first ID. To obtain the values of the
parameters of all elite configurations found by \irace use:
<<get_elites, prompt=TRUE, eval=TRUE, comment="">>=
logfile <- system.file(package="irace", "exdata", "irace-acotsp.Rdata", mustWork=TRUE)
getFinalElites(logfile, n = 0)
@
\item \code{iterationElites}: A vector containing the best candidate configuration ID of each iteration. The best configuration found corresponds to the last one
of this vector.
<<show_iditelites, prompt=TRUE, eval=TRUE, comment="">>=
print(iraceResults$iterationElites)
@
One can obtain the full configuration with:
<<get_elite, prompt=TRUE, eval=TRUE, comment="">>=
last <- length(iraceResults$iterationElites)
id <- iraceResults$iterationElites[last]
getConfigurationById(iraceResults, ids = id)
@
\item \code{rejectedConfigurations}: A vector containing the rejected configurations IDs. These
correspond to configurations that produced failed executions and were ignored by \irace during the
configuration process. See \autoref{sec:reject} to enable the detection of such configurations.
\item \code{experiments}: A matrix with configurations as columns and instances
as rows. Column names correspond to the internal identifier of the configuration
(\code{allConfigurations$.ID.}). The results of a particular
configuration can be obtained using:
<<get_experiments, prompt=TRUE, eval=TRUE, comment="">>=
# As an example, we use the best configuration found
best_config <- getFinalElites(iraceResults, n = 1)
best_id <- as.character(best_config$.ID.)
# Obtain the results of the best configuration
all_exp <- iraceResults$experiments[, best_id]
# all_exp is a vector and names(all_exp) is the (instance,seed) index.
all_exp
# Obtain the results of the first and best configurations
all_exp <- iraceResults$experiments[, c("1", best_id)]
# all_exp is a matrix: colnames(all_exp) is configurationID and
# rownames(all_exp) is the (instance,seed) index.
all_exp
@
When a configuration was not executed on an instance, its value is \code{NA}. A
configuration may not be executed on an instance because: (1) it was not created
yet when the instance was used, or (2) it was
discarded by the statistical test and not executed on subsequent instances, or
(3) the race terminated before this instance was considered.
Row names correspond to the row index of the \code{(instance, seed)} pairs
in the data frame returned by \code{get_instanceID_seed_pairs()}. The instanceID and seed used
for a particular experiment can be obtained with:\sloppy
<<get_instance_seed, prompt=TRUE, eval=TRUE, comment="">>=
# As an example, we get instanceID, seeds and instances of the experiments
# of the best configuration.
# We could get the indexes of the instances on which at least one
# configuration was executed:
pair_index <- which(apply(!is.na(all_exp), 1L, any))
# or the instances on which all configurations were executed:
pair_index <- which(apply(!is.na(all_exp), 1L, all))
# but in this example we get the indexes of the instances executed for
# the best configuration.
pair_index <- which(!is.na(all_exp[, best_id]))
instanceID <- get_instanceID_seed_pairs(iraceResults)[["instanceID"]][pair_index]
# or get the seeds
get_instanceID_seed_pairs(iraceResults)[["seed"]][pair_index]
# or obtain the actual instances.
iraceResults$scenario$instances[instanceID]
# If the instances are of atomic type (integers, floating-point numbers or
# character strings), the above is similar to:
get_instanceID_seed_pairs(iraceResults, index = pair_index, instances=TRUE)
@
\item \code{experimentLog}: A matrix with columns \code{iteration, instance, configuration}. This matrix contains the log of all the experiments that \irace
performs during its execution. The \code{instance} column refers to the index of the
data frame returned by \code{get_instanceID_seed_pairs()}. When \parameter{capping} is enabled a column \code{bound} is added to log the execution bound applied
for each execution.
\item \code{softRestart}: A logical vector that indicates if a soft restart was
performed on each iteration. If \code{FALSE}, then no soft restart was performed.
See option \parameter{softRestart} in \autoref{sec:irace options}.
\item \code{state}: A list that contains the state of \irace, the recovery
(\autoref{sec:recovery}) is done using the information contained in this object. The
probabilistic model of the last elite configurations can be found here by doing:
<<get_model, prompt=TRUE, eval=TRUE, comment="">>=
# As an example, we get the model probabilities for the
# localsearch parameter.
iraceResults$state$model["localsearch"]
# The order of the probabilities corresponds to:
iraceResults$scenario$parameters$domains$localsearch
@
The example shows a list that has one element per elite configuration (ID as
element name). In this case, \code{localsearch} is a categorical parameter and
it has a probability for each of its values.
\item \code{testing}: A list that contains the testing results.
The list contains the following elements:
\begin{itemize}
\item \code{experiments}: Matrix of experiments in the same format as the
\code{iraceResults$experiments} matrix. The column names indicate the candidate
configuration identifier and the row names contain the name of the instances.
<<get_test_exp, prompt=TRUE, eval=TRUE, comment="">>=
# Get the results of the testing
iraceResults$testing$experiments
@
\item \code{seeds}: The seeds used for the experiments, each seed corresponds to
each instance in the rows of the test \code{experiments} matrix.
<<get_test_seeds, prompt=TRUE, eval=TRUE, comment="">>=
# Get the seeds used for testing
iraceResults$testing$seeds
@
In the example, instance \code{1000-1.tsp} is executed with seed \Sexpr{iraceResults$testing$seeds[1]}.
\end{itemize}
\end{itemize}
\subsection{Analysis of results}\label{sec:analysis}
The final configurations returned by \irace are the elites of the final race.
They are reported in decreasing order of performance, that is, the best
configuration is reported first.
If testing is performed, you can further analyze the resulting best
configurations by performing statistical tests in \aR:
% FIXME: iraceplot
% or just plotting the results:
%<<plot_test, fig.pos="tbp", fig.align="center", fig.height = 4, fig.width = 8, out.width='0.85\\textwidth', fig.cap="Boxplot of the testing results of the best configurations.", prompt=TRUE, eval=TRUE, comment="">>=
<<wilcox_test,prompt=TRUE, eval=TRUE, comment="">>=
results <- iraceResults$testing$experiments
# Wilcoxon paired test
conf <- gl(ncol(results), # number of configurations
nrow(results), # number of instances
labels = colnames(results))
pairwise.wilcox.test (as.vector(results), conf, paired = TRUE, p.adj = "bonf")
@
% # Plot the results
% configurationsBoxplot (results, ylab = "Solution cost")
The Kendall concordance coefficient (\code{W}) and the Spearman's rho can be
applied over data that has the characteristics of the data obtained in the testing,
that is a full matrix where all configurations are executed in all instances. \code{W}
can show if the configurations tested have an homogeneous performance on the used instances
set. If evidence of an heterogeneous scenario found we recommend to make some adjustments
in the \irace options as described in \autoref{sec:het}.
<<conc, prompt=TRUE, eval=TRUE, comment="">>=
irace:::concordance(iraceResults$testing$experiments)
@
%
%% FIXME: iraceplot
% During the tuning, \irace iteratively updates the sampling models of the
% parameters to focus on the best regions of the parameter search space. The
% frequency of the sampled configurations can provide insights on the parameter
% search space. We provide a function for plotting the
% frequency of the sampling of a set of configurations. For more information on this function, please see the \aR help, type in the
% \aR console: \code{?parameterFrequency}. The following example
% plots the frequency of the parameters sampled during one \irace run:
% <<freq, fig.pos="tbp", fig.cap="Parameters sampling frequency.", out.width="\\textwidth",prompt=TRUE, eval=TRUE, comment="">>=
% parameterFrequency(iraceResults$allConfigurations, iraceResults$scenario$parameters)
% @
%% FIXME: iraceplot
% By using parallel coordinates plots, it is possible to analyze how the parameters
% interact with each other. For more information on this function, please see the \aR help, type
% in the \aR console: (\code{?parallelCoordinatesPlot}).
% The following example shows how to create a parallel
% coordinate plot of the configurations in the last two iterations of \irace.
% <<parcord, fig.pos="tbp", fig.align="center", out.width="0.7\\textwidth", fig.cap="Parallel coordinate plots of the parameters of the configurations in the last two iterations of a run of \\irace.", prompt=FALSE, eval=TRUE, comment="">>=
% # Get last iteration number
% last <- length(iraceResults$iterationElites)
% # Get configurations in the last two iterations
% conf <- getConfigurationByIteration(iraceResults,
% iterations = c(last - 1, last))
% parallelCoordinatesPlot (conf, iraceResults$scenario$parameters,
% param_names = c("algorithm", "alpha", "beta", "rho",
% "q0", "rasrank"),
% hierarchy = FALSE)
% @
%%% FIXME: This plot is completely misleading. We should either calculate the
%%% performance over the whole training set, or find a way to plot the
%%% performance estimated by irace as a confidence interval to account for
%%% different number of instances. Or simply plot a heatmap of instances x
%%% best-of-each iteration.
% It is also possible to plot the performance evolution of the best-so-far configuration over the number of experiments as follows:
% %%<<trainEvo, fig.pos="tbp", fig.align="center", out.width='0.75\\textwidth', fig.cap="Training set performance of the best-so-far configuration over number of experiments.", prompt=FALSE, eval=TRUE, comment="">>=
% # Get number of iterations
% iters <- unique(iraceResults$experimentLog[, "iteration"])
% # Get number of experiments (runs of target-runner) up to each iteration
% fes <- cumsum(table(iraceResults$experimentLog[,"iteration"]))
% # Get the mean value of all experiments executed up to each iteration
% # for the best configuration of that iteration.
% values <- sapply(iters, function(k) {
% instances <- as.character(
% unique(iraceResults$experimentLog[
% iraceResults$experimentLog[, "iteration"] == k,
% "instance"]))
% return(mean(iraceResults$experiments[
% instances,
% as.character(iraceResults$iterationElites[k])]))})
% plot(fes, values, type="s", xlab = "Number of runs of the target algorithm",
% ylab = "Estimated mean value over whole training set")
% points(fes, values)
% @
% FIXME: We should normalize the ranges per instance so since the performance
% on different instances is quite different.
It is also possible, as shown in Fig.~\ref{fig:testEvo}, to plot the performance on the test set of the best-so-far configuration over the number of experiments as follows:
<<testEvo, fig.pos="tb", fig.align="center", out.width='0.7\\textwidth', fig.cap="Testing set performance of the best-so-far configuration over number of experiments. Label of each point is the configuration ID.", prompt=FALSE, eval=TRUE, comment="">>=
# Get summary data from the logfile.
irs <- irace_summarise(iraceResults)
# Get number of iterations
iters <- irs$n_iterations
# Get number of experiments (runs of target-runner) up to each iteration
fes <- cumsum(table(iraceResults$state$experiment_log[["iteration"]]))
# Get the mean value of all experiments executed up to each iteration
# for the best configuration of that iteration.
elites <- as.character(iraceResults$iterationElites)
values <- colMeans(iraceResults$testing$experiments[, elites])
stderr <- function(x) sqrt(var(x)/length(x))
err <- apply(iraceResults$testing$experiments[, elites], 2L, stderr)
plot(fes, values, type = "s",
xlab = "Number of runs of the target algorithm",
ylab = "Mean value over testing set", ylim=c(min(values-err),max(values+err)))
points(fes, values, pch=19)
arrows(fes, values - err, fes, values + err, length=0.05, angle=90, code=3)
text(fes, values, elites, pos = 1)
@
The \irace package also provides an implementation of the ablation
method~\cite{FawHoos2015ablation}. See \autoref{sec:ablation}.
\medskip{}
Finally, more advanced visualizations of the behavior of \irace are provided by
the ACVIZ software package~\cite{DesRitLopPer2021acviz}, which is available at \url{https://github.com/souzamarcelo/acviz}. See an example in Fig.~\ref{fig:acviz}.
\begin{figure}[htb]
\centering
\includegraphics[width=\textwidth]{fig1u-acotsp-instances}
\caption{Visualization produced by ACVIZ~\cite{DesRitLopPer2021acviz}.}
\label{fig:acviz}
\end{figure}
%%
%%
%%
%% Advanced options
%%
%%
%%
\section{Advanced topics}
\subsection{Tuning budget}\label{sec:budget}
Before setting the budget for a run of \irace, please consider the number of
parameters that need to be tuned, available processing power and available
time. The optimal budget depends on the difficulty of the tuning scenario, the
size of the parameter space and the heterogeneity of the instances. Typical
values range from $1\,000$ to $100\,000$ runs of the target algorithm, although
smaller and larger values are also possible. Currently, \irace does not detect
whether the given budget allows generating all possible configurations. In such
a case, the use of \emph{iterated} racing is unnecessary: One can simply
perform a single race of all configurations (see FAQ in
\autoref{faq:allconfs}).
\Irace provides two options for setting the total tuning budget (\parameter{maxExperiments} and
\parameter{maxTime}). The option \parameter{maxExperiments}
limits the number of executions of \parameter{targetRunner} performed by \irace. The option \parameter{maxTime} limits the
total time of the \parameter{targetRunner} executions. When this latter option is used, \parameter{targetRunner}
must return the evaluation cost together with the execution time (\code{"cost time"}).
\begin{xwarningbox}
When the goal is to minimize the computation time of an algorithm, and you wish to use \parameter{maxTime} as the tuning budget, \parameter{targetRunner} must return the time also as the evaluation cost, that is, return the time twice as \code{"time time"}.
\end{xwarningbox}
\begin{xwarningbox}
When using \parameter{targetEvaluator} and using \parameter{maxTime} as tuning budget, \parameter{targetRunner} just returns the time (\code{"time"}) and \parameter{targetEvaluator} returns the cost.
\end{xwarningbox}
When using \parameter{maxTime}, \irace estimates the execution time of each \parameter{targetRunner} execution
before the configuration. The amount of budget used for the estimation is set with the option \parameter{budgetEstimation}
(default is $2\%$). The obtained estimation is adjusted after each iteration using the obtained results and it is used to estimate the number of experiments that can be executed. Internally, \irace uses the number of remaining experiments to
adjust the number of configurations tested in each race.
\subsection{Multi-objective tuning}\label{sec:multi objective}
Currently, \irace only optimizes one cost value at a time, which can be
solution cost, computation time or any other objective that is returned to
\irace by the \parameter{targetRunner}. If the target algorithm is
multi-objective, it will typically return not a single cost value, but a set of
objective vectors (typically, a Pareto front). For tuning such a target
algorithm with \irace, there are two alternatives. If the algorithm returns a
single vector of objective values, they can be aggregated into one single number
by using, for example, a weighted sum. Otherwise, if the target algorithm
returns a set of objective vectors, a unary quality metric (\eg~the
hypervolume) may be used to evaluate the quality of the set.\footnote{An implementation is publicly available at \url{http://lopez-ibanez.eu/hypervolume}~\cite{FonPaqLop06:hypervolume}}
% The first option is simple, it requires to devise a formula that can aggregate
% the objectives in a way that balances the importance of all of them. This might
% not be an easy task in some scenarios, and therefore using a more adequate
% indicator to evaluate the performance of a multi-objective optimizer, such as
% the hypervolume, is strongly advised.
%For using the hypervolume as evaluation \irace needs to postpone the evaluation of the
%configurations in an instance until all the executions have been completed. Once
%the executions are finalized, the evaluation starts by obtaining reference points for
%each objective from the configuration results. These points are used to define the
%Pareto front and the hypervolume of each configuration
%is calculated and the tuning process continues normally.
The use of aggregation or quality metrics often requires normalizing the
different objectives. If normalization bounds are known a priori for each
instance, normalized values can be computed by \parameter{targetRunner}.
Otherwise, the bounds may be dynamically computed while running \irace, by
using \parameter{targetEvaluator}. In this case, \parameter{targetRunner} will
save the output of the algorithm, then the first call
to \parameter{targetEvaluator} will examine the output produced by all calls
to \parameter{targetRunner} for the same instance, update the normalization
bounds and return the normalized output. Subsequent calls
to \parameter{targetEvaluator} for the same instance will simply return the
normalized output.
%
A similar approach can be used to dynamically compute the reference points or
sets often required by unary quality metrics.
For more information about defining a \parameter{targetEvaluator}, see
\autoref{sec:evaluator}. Examples of tuning a multi-objective target algorithm
using the hypervolume can be found in the examples at
\IRACEHOME{/examples/hypervolume} and \IRACEHOME{/examples/moaco}.
\subsection{Tuning for minimizing computation time}
\label{sec:capping}
When using \irace for tuning algorithms that only report computation time to
reach a target, \parameter{targetRunner} should return the execution time of a configuration instead of solution cost. When using \parameter{maxTime} as the budget, this means that \parameter{targetRunner} must return twice the execution time since the first value is the minimization objective and the second value is used to track the budget consumed.
Starting from version $3.0$, \irace includes an elitist racing procedure that
implements an \textbf{adaptive capping mechanism} \citep{PerLopHooStu2017:lion}.
Adaptive capping \citep{HutHooLeyStu2009jair} is a configuration technique
that avoids the execution of long runs of the target algorithm, focusing the
configuration budget in the evaluation of the best configurations found. This is
done by bounding the execution time of each configuration based on the best
performing candidate configurations.
To use adaptive capping, the \parameter{capping} option must be enabled and the
\parameter{elitist} irace option must be selected. When evaluating candidate
configurations on an instance, \irace calculates an execution bound based on the
execution times of the elite configurations. The \parameter{boundType} option defines
how the performance of the elite configurations is defined to obtain the execution bound.
The default value of \parameter{boundType} calculates the performance ($p_i^s$) of each
elite configuration ($s$) as the mean execution time of the instances already executed
in the race and the currently executed instance ($i$). The \parameter{cappingType} option
specifies the measure used to obtain the elite configurations bound. By default, the execution bound
is calculated as the median of the execution times of the elite configurations:
%
\begin{equation}
b_i= \text{Median}_{\theta_s \in \Celite} \{ p_i^s\}
\end{equation}
The execution bound for new configurations ($j$) is calculated by multiplying the elite configurations
bound by the number of instances ($i$) in the execution list and subtracting the mean execution time
of the instances executed by the candidate:
%
\begin{equation}
k_i^{'j} = b_{i} \cdot i + \bmin - p^{j}_{i-1} \cdot (i-1)
\end{equation}
A small constant \bmin is added to account for time measurements errors.
These settings are also used to apply a dominance elimination criterion together with the
statistical test elimination. The domination criterion is defined as:
%
\begin{equation}\label{eq:dom}
b_i + \bmin < p_i^{j}
\end{equation}
When elite configurations dominate new configurations, these are eliminated from the race.
\begin{xwarningbox}
The default statistical test when \parameter{capping} is enabled is \code{t-test}.
This test is more appropriate to configure algorithms for optimizing runtime (see \autoref{sec:stat test}).
\end{xwarningbox}
The execution bound is constantly adjusted by \irace based on the best configurations
times, nevertheless, a maximum execution time (\bmax) is never exceeded. This maximum
execution time must be defined in the configuration scenario when \parameter{capping}
is enabled. To specify the maximum execution bound for the target runner executions
use the \parameter{boundMax} option. The final execution bound ($k_i^j$) is calculated by:
%
\begin{equation}\label{eq:cap}
k_i^j = \begin{cases}
\bmax &\text{if $k_i^{'j} > \bmax$,} \\
\min \{b_i, \bmax\}& \text{if $k_i^{'j} \leq 0$,}\\
k_i^{'j}&\text{otherwise;}
\end{cases}
\end{equation}
Additionally, the \parameter{boundDigits} option defines the precision of the time bound
provided by \irace, the default setting is 0.
Timed out executions occur when the maximum execution bound
(\parameter{boundMax}) is reached and the algorithm has not achieved successful
termination or a defined quality goal. In this case, it is a common practice
to apply a penalty known as PARX, in which timeouts are penalized by
multiplying \parameter{boundMax} by a constant $X$. The constant $X$ may be set
using the \parameter{boundPar} option. Bounded executions are executions that
do not achieve successful termination or a defined quality goal in the
execution bound ($k_i^j$) set by \irace, which is smaller than
\parameter{boundMax}. The \parameter{boundAsTimeout} option replaces the
evaluation of bounded executions by the \parameter{boundMax} value. More
details about the implementation of adaptive capping can be found in
\citet{PerLopHooStu2017:lion}.
\begin{xwarningbox}
Note that bounded executions are not timed out executions and thus, they will not be penalized by PARX.
\end{xwarningbox}
\begin{xwarningbox}
Penalized evaluations of timed out and bounded executions are only used for
the elimination tests and the comparison between the quality of
configurations. To calculate execution bounds and computation budget
consumed, \irace uses only unpenalized execution times. The unpenalized
execution time must be provided by the target runner or target evaluator as
described in \autoref{sec:runner} and \autoref{sec:evaluator} .
\end{xwarningbox}
\begin{xwarningbox}
More advanced capping methods that are applicable to minimizing solution cost are available when combining irace with the \texttt{capopt} package described by \citet{SouRitLop2021cap}.
\end{xwarningbox}
\subsection{Hyper-parameter optimization of machine learning methods}
The \irace package can also be used for model selection and hyper-parameter
optimization of machine learning (ML) methods. We will next explain a possible
setup for one given dataset and using $10$-fold cross-validation
(CV). Generalizing to multiple datasets and different resampling strategies,
e.g. leave-one-out, is straightforward.
First, split the dataset into training, to be used by \irace, and testing, to
be used for evaluating the performance of the configuration returned by
\irace. A typical split could be $70\%$ and $30\%$, respectively.
The training set is used by \irace to perform $10$-fold CV, that is, the data
is split into $10$ folds. A single run of the \parameter{targetRunner} will use
$9$ folds for training and the remaining fold for validation. Splitting the
data into folds can be done at each call of \parameter{targetRunner} or before
running \irace, however, it is important that the split is always the same for
every call of the \parameter{targetRunner}, i.e., the content of the folds does
not change, only which folds are used for training and validation will change.
The setup of \irace should be as follows:
%
\begin{itemize}
\item \parameter{trainInstancesFile}\code{="train-instances.txt"}, where this
file contains one number per line from $1$ to $10$. This number will tell the
\parameter{targetRunner} which fold should be used for validation.
\item \parameter{trainInstancesDir}\code{=""}, because the folds are the
``instances'' and you do not have actual instance files. If you want to pass
the name of the dataset to the \parameter{targetRunner}, you can specify it
either at each line of \code{"train-instances.txt"}, directly in the
\parameter{targetRunner}, or as a fixed parameter in the
\parameter{parameterFile}.
\item \parameter{deterministic}\code{=1} unless it really makes sense to
train more than once the same ML model on the same data. If it makes sense,
then your \parameter{targetRunner} should use the seed passed by \irace to
seed the ML model before training.
\item \parameter{sampleInstances}\code{=0} because the folds should already be generated by randomly sampling the dataset.
\item \parameter{testType}\code{="t-test} because the performance metrics in
ML are typically the mean of the CV results, which assumes that the
performance are close to normally distributed.
\item \parameter{firstTest}\code{=2} because \irace should discard
configurations very aggressively looking for maximum generality.
\end{itemize}
\noindent%
Finally, your \parameter{targetRunner} needs to be able to do the following:
\begin{itemize}
\item Receive from \irace the hyper-parameter settings, the dataset name and a
fold number (the ``instance''). Let us use fold 3 as an example.
\item Train the ML model on the whole training set minus fold 3, then validate
(score) the model on fold 3 and return the score to \irace (negated if the
score must be maximized, because \irace assumes minimization). Since each
fold is different, each instance should give a different result. Each row in
the table printed by \irace should print something different; otherwise,
something is wrong in your setup.
\end{itemize}
The above is actually 10 times faster than doing 10-fold CV for each call to
\parameter{targetRunner}, thus, you should assign to \irace 10 times the budget
than what would be assigned to other methods that do a complete 10-fold CV at
each step.
%% FIXME: They removed the tutorial
%% For such a task, we recommend the \pkg{mlr} package~\citep{R:mlr}.
%% The following webpage documents how to use \irace for this purpose: \url{https://mlr-org.github.io/mlr-tutorial/devel/html/advanced_tune/index.html}
\subsection{Heterogeneous scenarios}
\label{sec:het}
We classify a scenario as homogeneous when the target algorithm has a
consistent performance regarding the instances; roughly speaking, good
configurations tend to perform well and bad configurations tend to perform
poorly on all instances of the problem. By contrast, in heterogeneous
scenarios, the target algorithm has an inconsistent performance on different
instances, that is, some configurations perform well for a subset of the
instances, while they perform poorly for a different subset.
When facing a heterogeneous scenario, the first question should be whether the
objective of tuning is to find configurations that perform reasonably well over
all instances, even if that configuration is not the best ones in any
particular instance (a generalist). If this is not the goal, then it would be
better to partition instances into more similar subsets and execute \irace
separately on each subset. This will lead to a portfolio of algorithm
configurations, one for each subset, and algorithm selection techniques can be
used to select the best configuration from the portfolio when facing a new
instance.
To make sure \irace is not misled by results on few instances, it may be useful
to increase the number of instances executed before doing a statistical test
using the option \parameter{firstTest}, \eg \code{--first-test 10} (default
value is 5), in order to see more instances before discarding configurations.
The option \parameter{elitistNewInstances} in elitist \irace (option
\parameter{elitist}) can be used to increase the number of new instances
executed in each iteration, \eg \code{--elitist-new-instances 5} (default value
is 1).
If finding an overall good configuration for all the instances is the
objective, then we recommend that instances are randomly sampled (option
\parameter{sampleInstances}), unless one can provide the instances in a
particular order that does not bias the tuning towards any subset.
If instances are easily categorized in different classes, then we recommend to
create ``blocks'' of instances in \parameter{trainInstancesFile}, where each
block should contain one instance from each class. Then set the option
\parameter{blockSize} to the number of classes within each block, so that
\irace will always see a complete block of instances before eliminating
configurations. The value of \parameter{blockSize} will multiply the values of
\parameter{firstTest}, \parameter{eachTest} and
\parameter{elitistNewInstances}. Randomly sampling instances
(\parameter{sampleInstances}\code{=1}) will randomly sample the blocks but not
break the blocks.
While executing \irace, the homogeneity of the scenario can be observed by
examining the values of Spearman's rank correlation coefficient and Kendall's
concordance coefficient in the text output of \irace. See \autoref{sec:output
text} for more information.
\subsection{Choosing the statistical test}
\label{sec:stat test}
The statistical test used in \irace identifies statistically bad performing
configurations that can be discarded from the race in order to save
budget. Different statistical tests use different criteria to compare the
cost of the configurations, which has an effect on the tuning results.
\Irace provides two types of statistical tests
(option \parameter{testType}). Each test has different characteristics that are
beneficial for different goals:
%
\begin{itemize}
\item Friedman test (\code{F-test}): This test uses the ranking of the
configurations to analyze the differences between their performance. This
makes the test suitable for scenarios where the scale of the performance
metric is not as important to assess configurations as their relative
ranking. This test is also indicated when the distribution of the mean
performances deviates greatly from a normal distribution. For example, the
ranges of the performance metric on different instances may be completely
difference and comparing the performance of different configurations using
the mean over multiple instances may be deceiving. We recommend to use the
\code{F-test} (default when \parameter{capping} is not enabled) when tuning
for solution cost and whenever the best performing algorithm should be among
the best in as many instances as possible.
\item Student's t-test (\code{t-test}): This test uses the mean performance of
the configurations to analyze the differences between the
configurations.\footnote{The t-test does not require that the performance
values follow a normal distribution, only that the distribution of sample
means does. In practice, the t-test is robust despite large deviations from
the assumptions.} This makes the test suitable for scenarios where the
differences between values obtained for different instances are relevant to
assess good configurations. We recommend using t-test, in particular, when
the target algorithm is minimizing computation time and, in general, whenever
the best configurations should obtain the best average solution cost.
\end{itemize}
The confidence level of the tests may be adjusted by using the
option \parameter{confidence}. Increasing the value of \parameter{confidence}
leads to a more strict statistical test. Keep in mind that a stricter test
will require more budget to identify which configurations perform worse. A less
strict test discards configurations faster by requiring less evidence against
them and, therefore, it is more likely to discard good configurations.
\subsection{Complex parameter space constraints}\label{sec:complex_domains}
Some parameters may have complex dependencies. Ideally, parameters should be
defined in the way that is more likely to help the search performed by
\irace. For example, when tuning a branch and bound algorithm, one may have the
following parameters:
\begin{itemize}
\item branching (\code{b}) that takes values in \code{\{0,1,2,3\}}, where 0 indicates no branching will be used
and the rest are different types of branching.
\item stabilization (\code{s}) that takes values in \code{\{0,1,2,3,4,5,6,7,8,9,10\}}, of which for \code{b=0}
only \code{\{0,1,2,3,4,5\}} are relevant.
\end{itemize}
In this case, it is not possible to describe the parameter space by defining
only two parameters for \irace. An extra parameter must be introduced as
follows:
\begin{center}
\begin{minipage}{0.8\linewidth}
\begin{CodeInput}
# name label type range condition
b "-b " c (0,1,2,3)
s1 "-s " c (0,1,2,3,4,5) | b == "0"
s2 "-s " c (0,1,2,3,4,5,6,7,8,9,10) | b != "0"
\end{CodeInput}
\end{minipage}
\end{center}
Parameters whose values depend on the value of other parameters may also
require using extra parameters or changing the parameters and processing them
in \parameter{targetRunner}. For example, given the following parameters:
%
\begin{itemize}
\item Population size (\code{p}) takes the integer values $[1,100]$.
\item Selection size (\code{s}) takes the same values but no more than the
population size, that is $[1,$\code{p}$]$.
\end{itemize}
In this case, it is possible to describe the parameters \code{p} and \code{s}
using surrogate parameters for \irace that represent a ratio of the original
interval as follows:
\begin{center}
\begin{minipage}{0.8\linewidth}
\begin{CodeInput}
# name label type range
p "-p " i (1,100)
s_f "-s " r (0.0,1.0)
\end{CodeInput}
\end{minipage}
\end{center}
%
and \parameter{targetRunner} must calculate the actual value of \code{s} as $\min(\max(\text{round}(\text{\code{s\_f}}\cdot \text{\code{p}}, 1)), 100)$. For
example, if the parameter \code{p} has value 50 and the surrogate parameter \code{s\_f} has value 0.3, then \code{s} will have value 15.
The processing within \parameter{targetRunner} can also split and join
parameters. For example, assume the following parameters:
\begin{center}
\begin{minipage}{0.8\linewidth}
\begin{CodeInput}
# name label type range
m "-m " i (1,250)
e "-e " r (0.0,2.0)
\end{CodeInput}
\end{minipage}
\end{center}
These parameters could be used to define a value
$\texttt{m}\cdot 10^\texttt{e}$ for another parameter
(\code{--strength}) not known by \irace. Then, \parameter{targetRunner} takes care of parsing
\code{-m} and \code{-e}, computing the strength value and passing the parameter
\code{--strength} together with its value to the target algorithm.
More complex parameter space constraints may be implemented by means of the
\parameter{repairConfiguration} function (\autoref{sec:repairconf}).
\subsection{Unreliable target algorithms and immediate rejection}\label{sec:reject}
There are some situations in which the target algorithm may fail to execute
correctly. By default, \irace stops as soon as a call
to \parameter{targetRunner} or \parameter{targetEvaluator} fails, which helps to
detect bugs in the target algorithm. Sometimes the failure cannot be fixed
because it is due to system problems, network issues, memory limits, bugs for
which no fix is available, or fixing them is impossible because there is no
access to the source code.
In those cases, if the failure is caused by random errors or transient system
problems, one may wish to ignore the error and try again the same call in the
hope that it succeeds. The option \parameter{targetRunnerRetries} indicates the
number of times a \parameter{targetRunner} execution is repeated if it
fails. Use this option only if you know additional repetitions could be
successful.
If the target algorithm consistently fails for a particular set of
configurations, these configurations may be declared as forbidden
(\autoref{sec:forbidden}) so that \irace avoids them. On the other hand, if
the configurations that cause the problem are unknown,
the \parameter{targetRunner} should return \code{Inf} so that \irace
immediately rejects the failing configuration. This immediate rejection should
be used with care according to the goals of the tuning. For example, a
configuration that crashes on a particular instance, \eg by running out of
memory, might still be considered acceptable if it gives very good results on
other instances. The configurations which were rejected during the execution of \irace are saved
in the Rdata output file (see \autoref{sec:output r}).
\begin{xwarningbox}
If the configuration budget is specified in total execution time (\parameter{maxTime} option),
immediate rejected executions must provide the cost and time (which must be \code{Inf 0}).
Nevertheless, rejected configurations will be excluded from the execution time estimation
and the execution bound calculation.
\end{xwarningbox}
\subsection{Ablation Analysis}\label{sec:ablation}
The ablation method~\cite{FawHoos2015ablation} takes two configurations (source
and target) and generates a sequence of configurations that differ between each
other just in one parameter, where parameter values in source are replaced by
values from target. The sequence can be seen as a ``path'' from the source to
the target configuration. This can be used to find new better ``intermediate''
configurations or to analyse the impact of the parameters in the performance.
To perform ablation, you can use the \code{ablation()} \aR function
%
<<ablation, prompt=FALSE, eval=FALSE>>=
ablog <- ablation("irace.Rdata", src = 1)
plotAblation(ablog)
@
%
\noindent or the \path{ablation} command-line executable (see more details below).
%
%<<linux_irace_help,engine='bash',eval=FALSE>>=
\begin{lstlisting}[style=BashInputStyle]
ablation -l irace.Rdata --src 1 --plot plot-ablation.pdf --plot-type rank,boxplot
\end{lstlisting}
%@
You may specify the IDs of the source and target configurations. By default, the
source is taken as the first configuration evaluated by \irace and the target
as the best overall configuration found. Use the function \code{plotAblation} to visualize the ablation results
(Fig.~\ref{fig:testAb}).
<<testAb, fig.pos="htb!", fig.align="center", out.width="0.75\\textwidth", fig.cap="Example of plot generated by \\code{plotAblation()}.", prompt=FALSE, eval=TRUE, echo=FALSE>>=
logfile <- system.file(package="irace", "exdata", "log-ablation.Rdata", mustWork=TRUE)
plotAblation(logfile)
@
The function returns a list containing the following elements:
%
\begin{description}
\item \code{configurations}: A dataframe of configurations tested during ablation.
\item \code{instances}: The instances used for the ablation.
\item \code{scenario}: Scenario options provided by the user.
\item \code{trajectory}: Best configuration IDs at each step of the ablation.
\item \code{best}: Best overall configuration found.
\end{description}
We also provide a command-line executable (\path{ablation.exe} in Windows) that allows you to perform ablation without launching \aR. It is installed in the same location as the \irace command-line executable and has the following options:
<<ablation_cmdline, prompt=FALSE, eval=TRUE,echo=FALSE, comment="">>=
ablation_cmdline("--help")
@
\subsection{Post-selection race}\label{sec:postselection}
At the end of the configuration process, \irace will automatically run a
post-selection race \citep{YuaStuMonLauBir13} to try to consume any remaining
budget. This post-selection race will not generate new configurations. It will
only do additional runs on configurations already evaluated in the current run
of \irace. First, it will try to evaluate the best configurations found in new
instances not seen yet. If this is not possible due to the budget available,
then it will evaluate configurations that have been evaluated on fewer
instances than the best configuration found. Currently, this automatic
post-selection is not supported when the budget is measured in runtime
(\parameter{maxTime}. The automatic post-selection can be disabled using the
option \parameter{postselection}.
It is also possible to manually run a post-selection race after completing a run of \irace using the R function \iracefun{psRace}.
<<postsel, prompt=FALSE, eval=FALSE>>=
# Execute all elite configurations in the iterations
psRace("irace.Rdata", max_experiments = 0.5, iteration_elites=TRUE)
# Execute a set of configurations IDs providing budget
psRace("irace.Rdata", conf_ids = c(34, 87, 102, 172, 293), max_experiments = 500)
@
The execution of the post-selection race adds an element (\code{psrace_log}) to
the \code{iraceResults} object saved in the \irace log file
(\parameter{logFile}). The post-selection log consists of a list with the
following elements:
\begin{description}
\item\code{configurations}: Configurations used in the post-selection race.
\item\code{instances}: Data frame with the instances and seeds used in the post-selection race.
\item \code{max_experiments}: Configuration budget assigned to the post-selection race.
\item \code{experiments}: Matrix of results generated by the post-selection
race, in the same format as the matrix
\code{iraceResults$experiments}. Column names are the configuration IDs and
row names are the instance IDs.
\item \code{elites}: Best configuration(s) found in the post-selection race.
\end{description}
\subsection[Parameter importance analysis using PyImp]{Parameter importance analysis using \PyImp}
The \PyImp\footnote{\url{https://github.com/automl/ParameterImportance}} tool developed by the AutoML group\footnote{\url{https://www.automl.org/}}
supports various parameter importance analysis methods using surrogate models.
Given a performance dataset of an algorithm configuration scenario, a Random Forest is built to predict performance of all algorithm configurations.
Parameter importance analyses are then applied on the prediction model.
The model serves as a surrogate for the original target algorithm, so that the algorithm does not need to be executed during the analyses.
Three analysis methods are supported, namely fANOVA~\citep{HutHooLey2014icml} (functional analysis of variance), forward selection~\cite{HutHooLey2013lion},
and ablation analysis with surrogates~\cite{BieLinEggFraFawHoo2017}.
Note that the \irace package directly supports ablation (without surrogate models) analysis with and without racing (\autoref{sec:ablation}).
Although ablation analysis without surrogates may be more time-consuming, results of the surrogate version may be less accurate than the non-surrogate one.
The \code{.Rdata} dataset generated by \irace can be used as input for \PyImp.
The package \code{irace2pyimp}\footnote{\url{https://github.com/ndangtt/irace2pyimp}} is able to convert an \code{irace.Rdata} file into the input format required by \PyImp. The conversion can be accessed either through the \aR console (function \code{irace2pyimp}), or via command line executable \code{irace2pyimp} (or \code{irace2pyimp.exe} in Windows) provided by the package. You can find the location of the executable by running this command in R: \Sexpr{'system.file(file.path("bin", "irace2pyimp"), package="irace2pyimp", mustWork=TRUE)'} or \Sexpr{'system.file(file.path("bin", "irace2pyimp.exe"), package="irace2pyimp", mustWork=TRUE)'} in Windows.
To see the usage of the executable, please run: \code{irace2pyimp --help}.
For more information on the \aR function \code{irace2pyimp}, type in the \aR console: \code{?irace2pyimp}.
Given as input an \code{irace.Rdata} file, the script will generate the following output files:
\begin{itemize}
\item \code{params.pcs}: a text file containing the parameter space definition.
\item \code{runhistory.json}: a JSON file containing the list of algorithm configurations evaluated during the tuning and the performance data obtained.
\item \code{traj_aclib2.json}: a JSON file containing the best configurations after each iteration of \irace. The last configuration will be used as the target configuration in ablation analysis.
\item \code{scenario.txt}: a text file containing the definition of the tuning scenario.
\item \code{instances.txt}: a text file containing the list of instances.
\item \code{features.csv}: a .csv file containing instance features. If no instance features are provided, the index of each instance will be used as a feature.
\end{itemize}
\PyImp can then be called using the files listed above as input. Several
examples on how to use the script and call \PyImp can be found at
\code{system.file("/examples/", package="irace2pyimp")}.
%\emph{\PyImp installation note:} At the time when this document is written (December 2019), the latest \PyImp release version (1.0.6) has some bugs.
%Please check on \url{https://pypi.org/project/PyImp/#history} to see if \PyImp version > 1.0.6 has been released.
%If it has, you can easily install \PyImp using \code{pip install pyimp}.
%Otherwise, please install \PyImp from source (\url{https://github.com/automl/ParameterImportance}), branch \code{development}. Installation instructions can be found at \url{https://automl.github.io/ParameterImportance/installation.html}.
%%
%% irace options
%%
\section{List of command-line and scenario options} \label{sec:irace options}
Most \irace options can be specified in the command line using a flag or in the
\irace scenario file using the option name (or setting their value in the
\code{scenario} list passed to the various \aR functions exported by the
package). This section describes the various \irace options that can be
specified by the user in this way.
\begin{xwarningbox}
Relative filesystem paths (e.g., \path{../scenario/}) given in the
command-line are relative to the current working directory (the directory at
which \irace is invoked). However, paths given in
the scenario file are relative to the directory containing
the scenario file. See also Table~\ref{tab:paths}.
\end{xwarningbox}
<<child-demo, child='section/irace-options.Rnw'>>=
@
%%
%%
%%
%% FAQ
%%
%%
%%
\section{FAQ (Frequently Asked Questions)}
\subsection{Is \irace minimizing or maximizing the output of my algorithm?}
By default, \irace considers that the value returned by \parameter{targetRunner} (or by
\parameter{targetEvaluator}, if used) should be \underline{\textbf{minimized}}. In case of a maximization
problem, one can simply multiply the value by -1 before returning it to
irace. This is done, for example, when maximizing the hypervolume (see the last
lines in \IRACEHOME{/examples/hypervolume/target-evaluator}).
\subsection{Are experiments with \irace reproducible?}
Short answer: Yes, under some conditions.\\
Long answer: According to the terminology described by \citet{LopBraPaq2021telo}, we define \emph{repeatability} as ``\emph{exactly repeating the original experiment, generating precisely the same results}''. Following this definition, a run of \irace is repeatable under the following conditions:
\begin{itemize}
\item Same version of \irace.
\item Same version of \aR (different versions of \aR may change the behavior of functions used by \irace).
\item The behavior of \parameter{targetRunner} is deterministic or exactly reproducible for the same instance, parameter configuration and random seed. Make sure that \parameter{targetRunner} uses the seed provided by \irace to initialize all random number generators used. If the result of \parameter{targetRunner} depends on CPU-time, wall-clock time or system load in any way, then \parameter{targetRunner} is not reproducible and neither will be \irace.
\item Same random seed (\parameter{seed}) given to \irace.
\item Same scenario options (\autoref{sec:irace options}). Although some options should not affect reproducibility (e.g., \parameter{debugLevel}), maintaining a list of such options will be a huge effort, thus the safest assumption is that any change may break reproducibility.
\item Same parameter space (\autoref{sec:target parameters}), including types, domains, conditions and forbidden configurations. The order of the parameters may also affect reproducibility (the name of the parameters should not) because it affects the order in which random numbers are used.
\item Same training instances provided and in the same order (\autoref{sec:training}). Even if the instances are sampled randomly (\parameter{sampleInstances}), a different initial order will produce a different sample even with the same random seed.
\item Same initial configurations (\autoref{sec:initial}), if any.
\end{itemize}
\subsection[Is it possible to configure a MATLAB algorithm with irace?]
{Is it possible to configure a \MATLAB algorithm with \irace ?}
Definitely. There are three main ways to achieve this:
\begin{enumerate}
\item Edit the \parameter{targetRunner} script to call \MATLAB in a non-interactive
way. See the \MATLAB documentation, or the following links.\footnote{\url{http://stackoverflow.com/questions/1518072/suppress-start-message-of-matlab}}\footnote{\url{http://stackoverflow.com/questions/4611195/how-to-call-matlab-from-command-line-and-print-to-stdout-before-exiting}} %
You would need to pass the parameter received by \parameter{targetRunner} to your \MATLAB script.\footnote{\url{https://www.mathworks.com/matlabcentral/answers/97204-how-can-i-pass-input-parameters-when-running-matlab-in-batch-mode-in-windows}}\footnote{\url{https://stackoverflow.com/questions/3335505/how-can-i-pass-command-line-arguments-to-a-standalone-matlab-executable-running}}
%
There is a minimal example in \IRACEHOME{/examples/matlab/}.
\item Call \MATLAB code directly from \aR using the
\pkg{matlabr} package (\url{https://cran.r-project.org/package=matlabr}). This
is a better option if you are experienced in \aR. Define \parameter{targetRunner} as
an \aR function instead of a path to a script. The function should
call your \MATLAB code with appropriate parameters.
\item Another possibility is calling \MATLAB directly from a different programming language and write \parameter{targetRunner} in that programming language, for example, in \proglang{Python} (see examples in \IRACEHOME{/examples/target-runner-python/}).\footnote{\url{https://www.mathworks.com/help/matlab/matlab_external/call-matlab-functions-from-python.html}\\
\url{https://www.mathworks.com/help/matlab/matlab_external/call-user-script-and-function-from-python.html}}
\end{enumerate}
\subsection{My program works perfectly on its own, but not when running under \irace. Is irace broken?}\label{faq:valgrind}
Every time this was reported, it was a difficult-to-reproduce bug, i.e., a \href{https://en.wikipedia.org/wiki/Heisenbug}{Heisenbug}, in
the program (target algorithm), not in \irace.
%
To detect such bugs, we recommend that you use, within
\parameter{targetRunner}, a memory debugger (e.g.,
\href{http://valgrind.org/}{\code{valgrind}}) to run your program. For example,
if your program is executed by \parameter{targetRunner} as:
%
\begin{lstlisting}[style=BashInputStyle]
${EXE} ${FIXED_PARAMS} -i ${INSTANCE} ${CONFIG_PARAMS} 1> ${STDOUT} 2> ${STDERR}
\end{lstlisting}
%
then replace that line with:
%
\begin{lstlisting}[style=BashInputStyle]
valgrind --error-exitcode=1 ${EXE} ${FIXED_PARAMS} -i ${INSTANCE} \
${CONFIG_PARAMS} 1> ${STDOUT} 2> ${STDERR}
\end{lstlisting}
If there are bugs in your program, they will appear in
\code{\${STDERR}}, thus do not delete those files. Memory debuggers will
significantly slowdown your code, so use them only as a means to find what is
wrong with your target algorithm. Once you have fixed the bugs, you should
remove the use of \code{valgrind}.
\subsection{\irace seems to run forever without any progress, is this a bug?}\label{faq:infloop}
Every time this problem was reported, the issue was in the target algorithm and
not in \irace. Some ideas for debugging this problem:
\begin{itemize}
\item Check that the target algorithm is really not running nor paused nor
sleeping nor waiting for input-output.
\item Use \parameter{debugLevel}\code{=3} to see how \irace calls \code{target-runner}, run the same command outside \irace and verify that it terminates.
\item Add some output to your algorithm that reports at the very end the runtime and exit code. Verify that this output is printed when \irace calls your algorithm.
\item In \code{target-runner}, print something to a log file \emph{after} calling your target algorithm. Verify that this output appears in the log file when \irace is running.
\item Set a maximum timeout when calling your target algorithm from \code{target-runner} (see FAQ~\ref{faq:timeout}).
\end{itemize}
\subsection{My program may be buggy and run into an infinite loop. Is it possible to set a maximum timeout?}\label{faq:timeout}
We are not aware of any way to achieve this using \aR. However, in
GNU/Linux, it is easy to implement by using the \code{timeout} command\footnote{\url{http://man7.org/linux/man-pages/man1/timeout.1.html}} in
\code{targetRunner} when invoking your program.
\subsection{When using the mpi option, \irace is aborted with an error message indicating that a
function is not defined. How to fix this?}\label{sec:mpi-error}
\pkg{Rmpi} does not work the same way when called from within a package and when called from a script or
interactively. When \irace creates the slave nodes, the slaves will load a copy of \irace automatically.
If the slave nodes are on different machines, they must have \irace installed. If \irace is not installed
system-wide, \aR needs to be able to find \irace on the slave nodes. This is usually done by setting \code{R_LIBS},
\code{.libPaths()} or by loading \irace using \code{library()} or \code{require()} with the argument ``\code{lib.loc}''.
The settings on the master are not applied to the slave nodes automatically, thus the slave
nodes may need their own settings. After spawning the slaves, it is too late to modify
those settings, thus modifying the shell variable \code{R_LIBS} seems the only valid
way to tell the slaves where to find \irace.
If the path is set correctly and the problem persists, please check these instructions:
%
\begin{enumerate}[leftmargin=*,widest=3]
\item Test that \irace and \pkg{Rmpi} work. Run \irace on a single machine (submit node), without calling \code{qsub}, \code{mpirun} or a
similar wrapper around \irace or \aR.
\item Test loading \irace on the slave nodes. However,
jobs submitted by \code{qsub}/\code{mpirun} may load \aR packages using a different mechanism from the way it happens if you log directly into the node (e.g., with \code{ssh}). Thus, you need to write a little \aR program such as:
<<faq3, eval=FALSE>>=
library(Rmpi)
mpi.spawn.Rslaves(nslaves = 10)
paths <- mpi.applyLB(1:10, function(x) {
library(irace); return(path.package("irace")) })
print(paths)
@
Submit this program to the cluster like you would submit \irace (using \code{qsub}, \code{mpirun} or whatever program is used to submit jobs to the cluster).
\item In the script \code{bin/parallel-irace-mpi}, the function \code{irace_main()} creates an MPI job for our cluster.
You may need to speak with the admin of your cluster and ask them how to best submit a job for MPI. There may be
some particular settings that you need. \pkg{Rmpi} normally creates log files; but \irace suppresses those files unless $\texttt{debugLevel} > 0$.
\end{enumerate}
Please contact us (\autoref{sec:contact}) if you have further problems.
\subsection[Error: 4 arguments passed to {.Internal(nchar)} which requires 3]{Error: 4 arguments passed to \code{.Internal(nchar)} which requires 3}
This is a bug in \aR 3.2.0 on Windows. The solution is to update your version of
\aR.
\subsection[Warning: In read.table(filename, header = TRUE, colClasses = "character", : incomplete final line found by \ldots]{Warning: \code{In read.table(filename,} \code{header = TRUE,} \code{colClasses =} \code{"character",} \code{: incomplete final line found by} \ldots}
This is a warning given by \aR when the last line of an input file does not
finish with the newline character. The warning is harmless and can be
ignored. If you want to suppress it, just open the file and press the
\code{ENTER} key at the end of the last line of the file to end the final line with a newline.
\subsection{How are relative filesystem paths interpreted by \irace?}\label{faq:relpaths}
The answer depends on where the path appears. Relative paths may appear as the
argument of command-line options, as the value of options given in the scenario
file, or within various scripts, functions or instance files. Table
\ref{tab:paths} summarizes how paths are translated from relative to absolute.
\begin{table}[h!]
\centering
\caption{Translation of relative to absolute filesystem paths.}
\label{tab:paths}
\begin{tabular}{rl}
\toprule
\bf Relative path appears as \ldots & \bf\ldots is relative to \ldots \\
\midrule
a string within \parameter{trainInstancesFile} & \parameter{trainInstancesDir}\\
a string within \parameter{testInstancesFile} & \parameter{testInstancesDir}\\
code within \parameter{targetRunner} or \parameter{targetEvaluator} & \parameter{execDir}\\
the value of \parameter{logFile} or \code{--log-file} & \parameter{execDir}\\
the value of other options in the scenario file & the directory containing the scenario file\\
the value of other command-line options & invocation (working) directory of \irace\\
\bottomrule
\end{tabular}
\end{table}
\subsection{My parameter space is small enough that \irace could generate all possible configurations; however, \irace generates repeated configurations and/or does not generate some of them. Is this a bug?}\label{faq:allconfs}
Typically, \irace is applied to parameter spaces that are much larger than what
can be explored within the budget given. Thus, \irace does not try to detect
whether all possible configurations can be evaluated for the given budget and
it does not waste computation time to check for repeated configurations. Thus,
if the parameter space is actually very small, the initial random sampling
performed by \irace may generate repeated configurations and/or never generate
some configurations, which is not ideal. If you still want to use
(non-iterated) racing, the recommended approach is to provide all
configurations explicitly to \irace (\autoref{sec:initial}) and execute a
single race (\parameter{nbIterations}\code{=1}) with exactly the number of
configurations provided (e.g., \parameter{nbConfigurations}\code{=240}). A
future version of \irace may automatically detect this case and switch to
non-iterated racing without having to set additional options. Future versions
may also implement computationally cheap checks for repeated
configurations.\footnote{If you are interested in implementing this, please
contact us!}
\subsection[On Windows and using target-runner.py (a Python file), I
get the error target-runner.py is not executable]{On Windows and using \code{target-runner.py} (a \proglang{Python} file), I
get the error ``\code{target-runner.py is not executable}''}\label{faq:py-not-exe}
The issue is that \code{.py} files are not executable on their own and you need \code{python.exe} to read the \code{.py} file and execute it. Linux knows how to do this if the first line of the file is ``\code{\#!/usr/bin/python}'', however, Windows doesn't know how to do it. In Windows you have 2 options:
\begin{itemize}
\item Create a \code{target-runner.bat} file that contains a line similar to (see \path{templates/windows/target-runner.bat}):
\begin{lstlisting}[style=BashInputStyle]
C:\path\to\python.exe C:\path\to\target-runner.py %instance% %seed% \
%candidate_parameters% 1>%stdout% 2>%stderr%
\end{lstlisting}
\item Or convert \code{target-runner.py} into an \code{.exe} file, for example, using \code{auto-py-to-exe}\footnote{\url{https://pypi.org/project/auto-py-to-exe/}}, so that you do not need a \code{.bat} file.
\end{itemize}
\subsection[Error in socketConnection(\ldots) : can not open the connection]{Error in \code{socketConnection("localhost", port = port, server = TRUE,} \code{lock = TRUE,} : can not open the connection}
This error may arise if you activate the \parameter{parallel} option of \irace and your \parameter{targetRunner} or \parameter{targetEvaluator} tries to setup a parallel cluster or execute code in parallel in a way that interacts badly with the parallel mechanism in \aR. In this case, you need to either investigate yourself if there is a way for the two parallel mechanisms to co-exist or, if that is not possible, disable parallelism in \irace or in your code. Note that packages or software used by your \parameter{targetRunner} may have a parallel mechanism enabled by default and unknown to you. This is definitely NOT a bug in \irace.
\subsection[irace does not print the call to the target-runner with --debug-level 2 when using the --parallel option]{\irace does not print the call to the \parameter{targetRunner} with \parameter{debugLevel}\code{=2} when using the \parameter{parallel}}
This is a limitation of Windows or Rstudio. Running without \parameter{parallel} should work. Unfortunately, we cannot fix this limitation in \irace. If you need to understand how \irace calls \parameter{targetRunner} when running in parallel, you can implement a logging mechanism able to handle parallelism directly inside the \parameter{targetRunner}.
%%
%%
%%
%% Contact info
%%
%%
%%
%FIXME: complete this section
%\section{Known problems}
\section{Resources and contact information} \label{sec:contact}
More information about the package can be found on the \irace webpage:
\begin{center} \url{https://iridia.ulb.ac.be/supp/IridiaSupp2016-003/index.html} \end{center}
For questions and suggestions please contact the development team through
the \irace package Google group:
\begin{center}
\url{https://groups.google.com/d/forum/irace-package}
\end{center}
or by sending an email to:
\begin{center}
\href{mailto:irace-package@googlegroups.com}{irace-package@googlegroups.com}
\end{center}
\section{Acknowledgements}
We would like to thank all the people that directly or indirectly have
collaborated in the development and improvement of \irace:
%
\newlist{acklist}{itemize*}{1}
\setlist[acklist]{label={}, before=\unskip{}, afterlabel=\unskip{ }, itemjoin={{,}}, itemjoin*={, and }, after={{.}}}
\begin{acklist}
\item Prasanna Balaprakash
\item Zhi (Eric) Yuan
\item Franco Mascia
\item Alberto Franzin
\item Anthony Antoun
\item Esteban Diaz Leiva %MATLAB example
\item Federico Caselli % bug reports with fixes
\item Pablo Valledor Pellicer % testing of --parallel under Windows
\item André de Souza Andrade
\item Nguyen Dang (\texttt{nttd@st-andrews.ac.uk}) % implemented \code{irace2pyimp}
\end{acklist}
\newcommand{\doi}[1]{doi:\hspace{.16667em plus .08333em}\discretionary{}{}{}\href{https://doi.org/#1}{\urlstyle{rm}\nolinkurl{#1}}}
\bibliographystyle{abbrvnat}
\ifthenelse {\boolean{Release}}{%
\bibliography{irace-package}%
}{%
\bibliography{optbib/abbrev,optbib/journals,optbib/authors,optbib/articles,optbib/biblio,optbib/crossref}%
}
\newpage
\begin{appendices}
\section{Installing R} \label{sec:installation}
This section gives a quick \aR installation guide that will work in
most cases. The official instructions are available at
\url{https://cran.r-project.org/doc/manuals/r-release/R-admin.html}
\subsection{GNU/Linux}
You should install \aR from your package manager. On a Debian/Ubuntu system
it will be something like:
\begin{lstlisting}[style=BashInputStyle]
sudo apt-get install r-base
\end{lstlisting}
Once \aR is installed, you can launch \aR from the Terminal and from the \aR prompt
install the \irace package (see \autoref{sec:irace install}).
\subsection{OS X}
You can install \aR directly from a CRAN mirror.\footnote{\url{https://cran.r-project.org/bin/macosx/}} %
Alternatively, if you use
homebrew, you can just brew the \aR formula from the science tap (unfortunately
it does not come already bottled so you need to have Xcode\footnote{Xcode download
webpage: \url{https://developer.apple.com/xcode/download/}} installed to compile it):
%<<R_OS_install,engine='bash',eval=FALSE>>=
\begin{lstlisting}[style=BashInputStyle]
brew tap homebrew/science
brew install r
\end{lstlisting}
%@
Once \aR is installed, you can launch \aR from the Terminal (or from your Applications),
and from the \aR prompt install the \irace package (see \autoref{sec:irace install}).
\subsection{Windows}
You can install \aR from a CRAN mirror.\footnote{\url{https://cran.r-project.org/bin/windows/}} %
We recommend that you
install \aR on a filesystem path without spaces, special characters or long
names, such as \path{C:\R}. Once \aR is installed, you can launch the \aR
console and install the \irace package from it (see \autoref{sec:irace
install}).
\section{targetRunner troubleshooting checklist} \label{sec:check list}
If the \parameter{targetRunner} script fails to return the output expected by
\irace, it can be sometimes difficult to diagnose where the problem lies. The
more descriptive errors provided by your script, the easier it will be to debug
it. If \parameter{targetRunner} enters an infinite loop, irace will wait
indefinitely (see FAQ in \autoref{faq:timeout}). If you are using temporary
files to redirect the output of your algorithm, check that these files are
properly created. We recommend to follow the structure of the example file
(\code{target-runner}) provided in \IRACEHOME{/templates}. The following
error examples are based on that example file.
In case of failure of \parameter{targetRunner}, \irace will print an error on
its output describing which execution of \parameter{targetRunner} failed.
%
Follow this checklist to detect where the problem is:
\begin{enumerate}[leftmargin=*,widest=9]
\item Make sure that your \code{targetRunner} script or program is at the specified location. If you see this error:
\begin{CodeInput}
Error: == irace == target runner './tuning/target-runner' does not exist
\end{CodeInput}
%
it means that \irace cannot find the \code{target-runner} file. Check that the file is at
the path specified by the error.
\item Make sure that your \code{targetRunner} script is an executable file and the user running \irace has permission to execute it. The following errors:
%
\begin{CodeInput}
Error: == irace == target runner './tuning/target-runner' is a directory,
not a file
\end{CodeInput}
or
\begin{CodeInput}
Error: == irace == target runner './tuning/target-runner' is not executable
\end{CodeInput}
%
mean that your \code{targetRunner} is not an executable file. In the first
case, the script is a folder and therefore there must be a problem with the
name of the script. In the second case, you must make the file executable,
which in GNU/Linux can be done by:
%
\begin{lstlisting}[style=BashInputStyle]
chmod +x ./tuning/target-runner
\end{lstlisting}
\item If your \code{targetRunner} script calls another program, make sure it is at the location described in the script
(variable \code{EXE} in the examples and templates). A typical output for such an error is:
%
\begin{CodeInput}
Error: == irace == running command ''./tuning/target-runner' 1 8 676651103
./tuning/Instances/1000-16.tsp --ras --localsearch 2 --alpha 4.03 --beta 1.89
--rho 0.02 --ants 37 --nnls 48 --dlb 0 --rasranks 15 2>\&1' had status 1
== irace == The call to target_runner_default was:
./tuning/target-runner 1 8 676651103 ./tuning/Instances/1000-16.tsp --ras
--localsearch 2 --alpha 4.03 --beta 1.89 --rho 0.02 --ants 37 --nnls 48
--dlb 0 --rasranks 15
== irace == The output was:
Tue May 3 19:00:37 UTC 2016: error: ./bin/acotsp: not found or not executable
(pwd: ./tuning/acotsp-arena)
\end{CodeInput}
%
You may test your script by copying the command line shown in the error and
executing \code{target-runner} directly on the execution directory (\parameter{execDir}). In this case, the command line is:
%
\begin{lstlisting}[style=BashInputStyle]
./tuning/target-runner 1 8 676651103 ./tuning/Instances/1000-16.tsp --ras \
--localsearch 2 --alpha 4.03 --beta 1.89 --rho 0.02 --ants 37 --nnls 48 \
--dlb 0 --rasranks 15
\end{lstlisting}
This executes the \code{targetRunner} script as \irace does. The output of this script must be only one number.
\item If your \code{targetRunner} file is an executable script in Python, \aR, Perl, Bash or some other interpreted programming language, make sure that the interpreter specified in the first line of the file exists at the correct location. For example, if the first line of \code{target-runner.py} is:
%
\begin{CodeInput}
#|/usr/bin/python
\end{CodeInput}
%
Then make sure that \path{/usr/bin/python} exists and it is executable. Otherwise, you will get an error such as:
%
\begin{CodeInput}
Error: == irace == error in running command
\end{CodeInput}
\item Check that your \code{targetRunner} script is actually returning one
number as output. For example:
%
\begin{CodeInput}
Error: == irace == The output of './tuning/target-runner 1 25 365157769
./tuning/Instances/1000-31.tsp --ras --localsearch 1 --alpha 0.26 --beta
6.95 --rho 0.69 --ants 56 --nnls 10 --dlb 0 --rasranks 7' is not numeric!
== irace == The output was:
Solution: 24479793
\end{CodeInput}
% FIXME: Duplicated above.
%% For testing your script, copy the command-line of \code{target-runner} and execute it directly on the execution directory (\parameter{execDir}):
%% %
%% \begin{lstlisting}[style=BashInputStyle]
%% ./tuning/target-runner 1 25 365157769 ./tuning/Instances/1000-31.tsp --ras \
%% --localsearch 1 --alpha 0.26 --beta 6.95 --rho 0.69 --ants 56 \
%% --nnls 10 --dlb 0 --rasranks 7
%% \end{lstlisting}
%% This executes the \code{targetRunner} script as \irace does.
In the example above, the output of \code{target-runner} is ``\code{Solution:
24479793}'', which is not a number. If \code{target-runner} is parsing the
output of the target algorithm, you need to verify that the code only parses
the solution cost value.
\item Check that your \code{targetRunner} script is creating the output files
for your algorithm. If you see an error as:
%
\begin{CodeInput}
== irace == The output was: Tue May 3 19:41:40 UTC 2016:
error: c1-9.stdout: No such file or directory
\end{CodeInput}
The output file of the execution of your algorithm has not been
created (check permissions) or has been deleted before the result can be read.
\item Other errors can produce the following output:
%
\begin{CodeInput}
== irace == The output was: Tue May 3 19:49:06 UTC 2016:
error: c1-23.stdout: Output is not a number
\end{CodeInput}
%
This might be because your \code{targetRunner} script is not executing your
algorithm correctly. To further investigate this issue, comment out the line
that eliminates the temporary files that saves the output of your
algorithm. Similar to this one
%
\begin{lstlisting}[style=BashInputStyle]
rm -f "${STDOUT}" "${STDERR}"
\end{lstlisting}
%
Execute directly the \code{targetRunner} command-line that is provided in the
error message, look in your execution directory for the files that are
created. Check the \code{.stderr} file for errors and the \code{.stdout} file
to see the output that your algorithm produces.
\item Some command within \code{targetRunner} may not be working correctly. In
that case, you must debug the commands individually exactly as \irace executes them. In
order to find where the problem is, print the commands to a log file before
executing them. For example:
%
\begin{lstlisting}[style=BashInputStyle]
echo "$EXE ${FIXED_PARAMS} -i $INSTANCE ${CONFIG_PARAMS}" >> ${STDERR}.log
$EXE ${FIXED_PARAMS} -i $INSTANCE ${CONFIG_PARAMS} 1> ${STDOUT} 2> ${STDERR}
\end{lstlisting}
%
then look at the \code{${STDERR}.log} file corresponding to the
\code{targetRunner} call that failed and execute/debug the last command
there.
\item If the language of your operating system, the \code{target-runner} or the
target algorithm is not English, \irace may not be able to recognize the
numbers generated by \code{target-runner}. We recommend that you run \irace,
the \code{target-runner} and the target algorithm under an English locale (or
make sure that their languages and number format are compatible).
\item It is possible that
\href{https://en.wikipedia.org/wiki/Heisenbug}{transient bugs} in the target
algorithm are only visible when running within \irace, and
all commands within \code{targetRunner} appear to work fine when executed directly in the
command-line outside \irace. See FAQ in \autoref{faq:valgrind}) for suggestions on how to detect such bugs.
\item If your \code{targetRunner} script works when running irace with \parameter{parallel}\code{=0} but it fails when using higher number of cores, this may be due to any number of reasons:
\begin{itemize}
\item If you submit jobs through a queuing system, the running environment
when using the queuing system may not be the same as when you launch \irace
yourself. The queuing system may also send the job to different machines
depending on the number of CPUs requested. One way to test this is to
submit the failing execution of \code{targetRunner} to the queuing system,
and specifically to any problematic machine.
\item When using MPI, some calls to \code{targetRunner} may run on different
computers than the one running the master \irace process. See FAQ in
\autoref{sec:mpi-error}.
\item Does \code{targetRunner} read or create intermediate files? These files
may cause a race condition when two calls to \code{targetRunner} happen at
the same time. You have to make sure that parallel runs of
\code{targetRunner} do not interfere with each other's files.
\item Maybe these files consume too much memory or fill the filesystem when
there are simultaneous \code{targetRunner} calls? Moreover, queuing systems
have stricter limits for computing nodes than for the submit/host node.
\item Does the machine or the queuing system impose any limits on number of
processes or CPU/memory/filesystem usage per job? Such limits may only
trigger when more than one process is executed in parallel, killing the
\code{targetRunner} process before it has a chance to print anything
useful. In that case, \irace may not detect the the program finished
unexpectedly, only that the expected output was not printed.
\end{itemize}
\end{enumerate}
\section{targetEvaluator troubleshooting checklist} \label{sec:evaluator check list}
Even if \parameter{targetRunner} appears to work, the use of \parameter{targetEvaluator} may lead to other problems. The same checklist of \code{targetRunner} can be followed here. In addition, we list here other potential problems unique to \code{targetEvaluator}:
\begin{enumerate}
\item If \parameter{targetEvaluator} fails only in the second or later
iteration, this may because output files or data generated by a previous call
to \parameter{targetRunner} are missing. Elite configurations are never
re-executed on the same instance and seed pair, that is, \irace will call
only once \parameter{targetRunner} for each pair of configuration ID and
instance ID. However, \parameter{targetEvaluator} is always re-executed,
which takes into account any updated information (normalization bounds,
reference sets/points, best-known values, etc.). Thus, any files or data
generated by \parameter{targetRunner} for a given configuration must remain
available to \parameter{targetEvaluator} as long as that configuration is
alive. The list of alive configurations is passed
to \parameter{targetEvaluator}, which may decide then which data to keep or
remove.
\end{enumerate}
\section{Glossary}
\begin{description}
\item[Parameter tuning:] Process of searching good settings for the parameters
of an algorithm under a particular tuning scenario (instances, execution
time, etc.).
\item[Scenario:] Settings that define an instance of the tuning problem. These
settings include the algorithm to be tuned (target), budget for the execution
of the target algorithm (execution time, evaluations, iterations, etc.), set
of problem instances and all the information that is required to perform the
tuning.
\item[Target algorithm:] Algorithm whose parameters will be tuned.
\item[Target parameter:] Parameter of the target algorithm that will be tuned.
\item[\irace option:] Configurable option of \irace.
\item[Elite configurations:] Best configurations found so far by \irace. New
configurations for the next iteration of \irace are sampled from the
probabilistic models associated to the elite configurations. All elite
configurations are also included in the next iteration.
\item[\hypertarget{irace_home}{\path{$IRACE_HOME}:}] The filesystem path where \irace is
installed. You can find this information by opening an \aR console and
executing:
<<R_irace_home2, prompt=FALSE, eval=FALSE>>=
system.file(package = "irace")
@
\end{description}
\section{NEWS}
\RecustomVerbatimCommand{\VerbatimInput}{VerbatimInput}%
{fontsize=\footnotesize,
%
frame=lines, % top and bottom rule only
framesep=1em, % separation between frame and text
rulecolor=\color{Gray},
%
label=\fbox{\color{Black}NEWS},
labelposition=topline,
%
% commandchars=\|\(\), % escape character and argument delimiters for
% % commands within the verbatim
% commentchar=* % comment character
}
%% NEWS is latin1 but this file is utf8
\begingroup\inputencoding{utf8}%
% FIXME: This doesn't break long lines, find another alternative.
\VerbatimInput{../NEWS.md}%
\endgroup
\end{appendices}
\end{document}
%%% Local Variables:
%%% TeX-master: "irace-package.Rnw"
%%% End:
% LocalWords: iteratively