Package adams.flow.transformer

Class WekaExperimentEvaluation

  • All Implemented Interfaces:
    adams.core.AdditionalInformationHandler, adams.core.CleanUpHandler, adams.core.Destroyable, adams.core.GlobalInfoSupporter, adams.core.logging.LoggingLevelHandler, adams.core.logging.LoggingSupporter, adams.core.option.OptionHandler, adams.core.QuickInfoSupporter, adams.core.ShallowCopySupporter<adams.flow.core.Actor>, adams.core.SizeOfHandler, adams.core.Stoppable, adams.core.StoppableWithFeedback, adams.core.VariablesInspectionHandler, adams.event.VariableChangeListener, adams.flow.core.Actor, adams.flow.core.ErrorHandler, adams.flow.core.InputConsumer, adams.flow.core.OutputProducer, Serializable, Comparable
    public class WekaExperimentEvaluation
extends adams.flow.transformer.AbstractTransformer
    Generates evaluation output of an experiment that was run previously.

    Input/output:
    - accepts:
       weka.experiment.Experiment
       weka.core.Instances
    - generates:
       java.lang.String


    -logging-level <OFF|SEVERE|WARNING|INFO|CONFIG|FINE|FINER|FINEST> (property: loggingLevel)
    The logging level for outputting errors and debugging output.
    default: WARNING
 
    -name <java.lang.String> (property: name)
    The name of the actor.
    default: WekaExperimentEvaluation
 
    -annotation <adams.core.base.BaseAnnotation> (property: annotations)
    The annotations to attach to this actor.
    default: 
 
    -skip <boolean> (property: skip)
    If set to true, transformation is skipped and the input token is just forwarded 
    as it is.
    default: false
 
    -stop-flow-on-error <boolean> (property: stopFlowOnError)
    If set to true, the flow gets stopped in case this actor encounters an error;
     useful for critical actors.
    default: false
 
    -silent <boolean> (property: silent)
    If enabled, then no errors are output in the console.
    default: false
 
    -tester <weka.experiment.Tester> (property: tester)
    The testing algorithm to use for performing the evaluations.
    default: weka.experiment.PairedCorrectedTTester -R 0 -S 0.05
 
    -comparison <ELAPSED_TIME_TRAINING|ELAPSED_TIME_TESTING|USERCPU_TIME_TRAINING|USERCPU_TIME_TESTING|SERIALIZED_MODEL_SIZE|SERIALIZED_TRAIN_SET_SIZE|SERIALIZED_TEST_SET_SIZE|NUMBER_OF_TRAINING_INSTANCES|NUMBER_OF_TESTING_INSTANCES|NUMBER_CORRECT|NUMBER_INCORRECT|NUMBER_UNCLASSIFIED|PERCENT_CORRECT|PERCENT_INCORRECT|PERCENT_UNCLASSIFIED|KAPPA_STATISTIC|MEAN_ABSOLUTE_ERROR|ROOT_MEAN_SQUARED_ERROR|RELATIVE_ABSOLUTE_ERROR|ROOT_RELATIVE_SQUARED_ERROR|CORRELATION_COEFFICIENT|SF_PRIOR_ENTROPY|SF_SCHEME_ENTROPY|SF_ENTROPY_GAIN|SF_MEAN_PRIOR_ENTROPY|SF_MEAN_SCHEME_ENTROPY|SF_MEAN_ENTROPY_GAIN|KB_INFORMATION|KB_MEAN_INFORMATION|KB_RELATIVE_INFORMATION|TRUE_POSITIVE_RATE|NUM_TRUE_POSITIVES|FALSE_POSITIVE_RATE|NUM_FALSE_POSITIVES|TRUE_NEGATIVE_RATE|NUM_TRUE_NEGATIVES|FALSE_NEGATIVE_RATE|NUM_FALSE_NEGATIVES|IR_PRECISION|IR_RECALL|F_MEASURE|MATTHEWS_CORRELATION_COEFFICIENT|AREA_UNDER_ROC|AREA_UNDER_PRC|WEIGHTED_TRUE_POSITIVE_RATE|WEIGHTED_FALSE_POSITIVE_RATE|WEIGHTED_TRUE_NEGATIVE_RATE|WEIGHTED_FALSE_NEGATIVE_RATE|WEIGHTED_IR_PRECISION|WEIGHTED_IR_RECALL|WEIGHTED_F_MEASURE|WEIGHTED_MATTHEWS_CORRELATION_COEFFICIENT|WEIGHTED_AREA_UNDER_ROC|WEIGHTED_AREA_UNDER_PRC> (property: comparisonField)
    The field to base the comparison of algorithms on.
    default: PERCENT_CORRECT
 
    -significance <double> (property: significance)
    The significance level (0-1).
    default: 0.05
    minimum: 1.0E-4
    maximum: 0.9999
 
    -test <int> (property: testBase)
    The index of the test base (normally the first classifier, ie '0').
    default: 0
    minimum: 0
 
    -row <adams.core.base.BaseString> [-row ...] (property: row)
    The list of fields that define a row (normally the dataset).
    default: Key_Dataset
 
    -col <adams.core.base.BaseString> [-col ...] (property: column)
    The list of fields that define a column (normally the schemes).
    default: Key_Scheme, Key_Scheme_options, Key_Scheme_version_ID
 
    -swap <boolean> (property: swapRowsAndColumns)
    If set to true, rows and columns will be swapped.
    default: false
 
    -format <weka.experiment.ResultMatrix> (property: outputFormat)
    The output format for generating the output.
    default: weka.experiment.ResultMatrixPlainText -mean-prec 2 -stddev-prec 2 -col-name-width 0 -row-name-width 25 -mean-width 0 -stddev-width 0 -sig-width 0 -count-width 5 -print-col-names -print-row-names -enum-col-names
 
    -header <boolean> (property: outputHeader)
    If set to true, then a header describing the experiment evaluation will 
    get output as well.
    default: true
    Author:
    fracpete (fracpete at waikato dot ac dot nz)
    See Also:
    Serialized Form

    • Field Summary

      Fields 
      Modifier and Type Field Description
      protected adams.core.base.BaseString[] m_Column
      the column (= classifiers).
      protected ExperimentStatistic m_ComparisonField
      the comparison field.
      protected weka.experiment.ResultMatrix m_OutputFormat
      the output format.
      protected boolean m_OutputHeader
      whether to output the header.
      protected adams.core.base.BaseString[] m_Row
      the row (= datasets).
      protected double m_Significance
      the significance.
      protected boolean m_SwapRowsAndColumns
      whether to swap rows and columns.
      protected int m_TestBase
      the test base.
      protected weka.experiment.Tester m_Tester
      the tester class to use.

      • Fields inherited from class adams.flow.transformer.AbstractTransformer

        BACKUP_INPUT, BACKUP_OUTPUT, m_InputToken, m_OutputToken

      • Fields inherited from class adams.flow.core.AbstractActor

        m_Annotations, m_BackupState, m_DetectedObjectVariables, m_DetectedVariables, m_ErrorHandler, m_Executed, m_Executing, m_ExecutionListeningSupporter, m_FullName, m_LoggingPrefix, m_Name, m_Parent, m_ScopeHandler, m_Self, m_Silent, m_Skip, m_StopFlowOnError, m_StopMessage, m_Stopped, m_StorageHandler, m_VariablesUpdated

      • Fields inherited from class adams.core.option.AbstractOptionHandler

        m_OptionManager

      • Fields inherited from class adams.core.logging.LoggingObject

        m_Logger, m_LoggingIsEnabled, m_LoggingLevel

      • Fields inherited from interface adams.flow.core.Actor

        FILE_EXTENSION, FILE_EXTENSION_GZ

    • Method Summary

      All Methods Instance Methods Concrete Methods 
      Modifier and Type Method Description
      Class[] accepts()
      Returns the class that the consumer accepts.
      String columnTipText()
      Returns the tip text for this property.
      String comparisonFieldTipText()
      Returns the tip text for this property.
      void defineOptions()
      Adds options to the internal list of options.
      protected List<String> determineColumnNames​(adams.core.base.BaseString[] list, String defaultList)
      Returns a vector with column names of the dataset, listed in "list".
      protected String doExecute()
      Executes the flow item.
      protected void evaluateExperiment​(weka.core.Instances data)
      Evaluates the experiment data.
      Class[] generates()
      Returns the class of objects that it generates.
      adams.core.base.BaseString[] getColumn()
      Returns the list of fields that identify a column.
      ExperimentStatistic getComparisonField()
      Returns the comparison field.
      protected weka.core.Instances getData​(weka.experiment.Experiment exp)
      Loads the experimental results.
      weka.experiment.ResultMatrix getOutputFormat()
      Returns the output format in use for generating the output.
      boolean getOutputHeader()
      Returns whether to output the header of the result matrix as well.
      String getQuickInfo()
      Returns a quick info about the actor, which will be displayed in the GUI.
      adams.core.base.BaseString[] getRow()
      Returns the list of fields that identify a row.
      double getSignificance()
      Returns the current significance level (0-1).
      boolean getSwapRowsAndColumns()
      Returns whether to swap rows and columns.
      int getTestBase()
      Returns the index of the test base.
      weka.experiment.Tester getTester()
      Returns the Tester in use.
      protected weka.experiment.Tester getTester​(weka.core.Instances data)
      Sets up the testing algorithm and returns it.
      String globalInfo()
      Returns a string describing the object.
      String outputFormatTipText()
      Returns the tip text for this property.
      String outputHeaderTipText()
      Returns the tip text for this property.
      String rowTipText()
      Returns the tip text for this property.
      void setColumn​(adams.core.base.BaseString[] value)
      Sets list of fields that identify a column.
      void setComparisonField​(ExperimentStatistic value)
      Sets the comparison field.
      void setOutputFormat​(weka.experiment.ResultMatrix value)
      Sets the output format to use for generating the output.
      void setOutputHeader​(boolean value)
      Sets whether to output the header of the result matrix as well.
      void setRow​(adams.core.base.BaseString[] value)
      Sets the list of fields that identify a row.
      void setSignificance​(double value)
      Sets the significance level (0-1).
      void setSwapRowsAndColumns​(boolean value)
      Sets whether to swap rows and columns.
      void setTestBase​(int value)
      Sets the index of the test base.
      void setTester​(weka.experiment.Tester value)
      Sets the Tester to use.
      String significanceTipText()
      Returns the tip text for this property.
      String swapRowsAndColumnsTipText()
      Returns the tip text for this property.
      String testBaseTipText()
      Returns the tip text for this property.
      String testerTipText()
      Returns the tip text for this property.

      • Methods inherited from class adams.flow.transformer.AbstractTransformer

        backupState, currentInput, execute, hasInput, hasPendingOutput, input, output, postExecute, restoreState, wrapUp

      • Methods inherited from class adams.flow.core.AbstractActor

        annotationsTipText, canInspectOptions, canPerformSetUpCheck, cleanUp, compareTo, configureLogger, destroy, equals, finalUpdateVariables, findVariables, findVariables, forceVariables, forCommandLine, forName, forName, getAdditionalInformation, getAnnotations, getDefaultName, getDetectedVariables, getErrorHandler, getFlowActors, getFlowExecutionListeningSupporter, getFullName, getName, getNextSibling, getParent, getParentComponent, getPreviousSibling, getRoot, getScopeHandler, getSilent, getSkip, getStopFlowOnError, getStopMessage, getStorageHandler, getVariables, handleError, handleException, hasErrorHandler, hasStopMessage, index, initialize, isBackedUp, isExecuted, isExecuting, isFinished, isHeadless, isStopped, nameTipText, performSetUpChecks, performVariableChecks, preExecute, pruneBackup, pruneBackup, reset, setAnnotations, setErrorHandler, setName, setParent, setSilent, setSkip, setStopFlowOnError, setUp, setVariables, shallowCopy, shallowCopy, silentTipText, sizeOf, skipTipText, stopExecution, stopExecution, stopFlowOnErrorTipText, updateDetectedVariables, updatePrefix, updateVariables, variableChanged

      • Methods inherited from class adams.core.option.AbstractOptionHandler

        cleanUpOptions, finishInit, getDefaultLoggingLevel, getOptionManager, loggingLevelTipText, newOptionManager, setLoggingLevel, toCommandLine, toString

      • Methods inherited from class adams.core.logging.LoggingObject

        getLogger, getLoggingLevel, initializeLogging, isLoggingEnabled

      • Methods inherited from interface adams.flow.core.Actor

        cleanUp, compareTo, destroy, equals, findVariables, getAnnotations, getDefaultName, getDetectedVariables, getErrorHandler, getFlowExecutionListeningSupporter, getFullName, getName, getNextSibling, getParent, getParentComponent, getPreviousSibling, getRoot, getScopeHandler, getSilent, getSkip, getStopFlowOnError, getStopMessage, getStorageHandler, getVariables, handleError, hasErrorHandler, hasStopMessage, index, isExecuted, isFinished, isHeadless, isStopped, setAnnotations, setErrorHandler, setName, setParent, setSilent, setSkip, setStopFlowOnError, setUp, setVariables, shallowCopy, shallowCopy, sizeOf, stopExecution, stopExecution, toCommandLine, variableChanged

      • Methods inherited from interface adams.core.AdditionalInformationHandler

        getAdditionalInformation

      • Methods inherited from interface adams.core.logging.LoggingLevelHandler

        getLoggingLevel, setLoggingLevel

      • Methods inherited from interface adams.core.logging.LoggingSupporter

        getLogger, isLoggingEnabled

      • Methods inherited from interface adams.core.option.OptionHandler

        cleanUpOptions, getOptionManager

      • Methods inherited from interface adams.core.VariablesInspectionHandler

        canInspectOptions

    • Field Detail

      • m_Tester

        protected weka.experiment.Tester m_Tester
        the tester class to use.

      • m_Significance

        protected double m_Significance
        the significance.

      • m_TestBase

        protected int m_TestBase
        the test base.

      • m_Row

        protected adams.core.base.BaseString[] m_Row
        the row (= datasets).

      • m_Column

        protected adams.core.base.BaseString[] m_Column
        the column (= classifiers).

      • m_SwapRowsAndColumns

        protected boolean m_SwapRowsAndColumns
        whether to swap rows and columns.

      • m_OutputFormat

        protected weka.experiment.ResultMatrix m_OutputFormat
        the output format.

      • m_OutputHeader

        protected boolean m_OutputHeader
        whether to output the header.

    • Constructor Detail

      • WekaExperimentEvaluation

        public WekaExperimentEvaluation()

    • Method Detail

      • globalInfo

        public String globalInfo()
        Returns a string describing the object.
        Specified by:
        globalInfo in interface adams.core.GlobalInfoSupporter
        Specified by:
        globalInfo in class adams.core.option.AbstractOptionHandler
        Returns:
        a description suitable for displaying in the gui

      • defineOptions

        public void defineOptions()
        Adds options to the internal list of options.
        Specified by:
        defineOptions in interface adams.core.option.OptionHandler
        Overrides:
        defineOptions in class adams.flow.core.AbstractActor

      • getQuickInfo

        public String getQuickInfo()
        Returns a quick info about the actor, which will be displayed in the GUI.
        Specified by:
        getQuickInfo in interface adams.flow.core.Actor
        Specified by:
        getQuickInfo in interface adams.core.QuickInfoSupporter
        Overrides:
        getQuickInfo in class adams.flow.core.AbstractActor
        Returns:
        null if no info available, otherwise short string

      • setTester

        public void setTester​(weka.experiment.Tester value)
        Sets the Tester to use.
        Parameters:
        value - the Tester

      • getTester

        public weka.experiment.Tester getTester()
        Returns the Tester in use.
        Returns:
        the Tester

      • testerTipText

        public String testerTipText()
        Returns the tip text for this property.
        Returns:
        tip text for this property suitable for displaying in the GUI or for listing the options.

      • setComparisonField

        public void setComparisonField​(ExperimentStatistic value)
        Sets the comparison field.
        Parameters:
        value - the field

      • getComparisonField

        public ExperimentStatistic getComparisonField()
        Returns the comparison field.
        Returns:
        the string

      • comparisonFieldTipText

        public String comparisonFieldTipText()
        Returns the tip text for this property.
        Returns:
        tip text for this property suitable for displaying in the GUI or for listing the options.

      • setSignificance

        public void setSignificance​(double value)
        Sets the significance level (0-1).
        Parameters:
        value - the significance

      • getSignificance

        public double getSignificance()
        Returns the current significance level (0-1).
        Returns:
        the significance

      • significanceTipText

        public String significanceTipText()
        Returns the tip text for this property.
        Returns:
        tip text for this property suitable for displaying in the GUI or for listing the options.

      • setTestBase

        public void setTestBase​(int value)
        Sets the index of the test base.
        Parameters:
        value - the index

      • getTestBase

        public int getTestBase()
        Returns the index of the test base.
        Returns:
        the index

      • testBaseTipText

        public String testBaseTipText()
        Returns the tip text for this property.
        Returns:
        tip text for this property suitable for displaying in the GUI or for listing the options.

      • setRow

        public void setRow​(adams.core.base.BaseString[] value)
        Sets the list of fields that identify a row.
        Parameters:
        value - the list of fields

      • getRow

        public adams.core.base.BaseString[] getRow()
        Returns the list of fields that identify a row.
        Returns:
        the list of fields

      • rowTipText

        public String rowTipText()
        Returns the tip text for this property.
        Returns:
        tip text for this property suitable for displaying in the GUI or for listing the options.

      • setColumn

        public void setColumn​(adams.core.base.BaseString[] value)
        Sets list of fields that identify a column.
        Parameters:
        value - the list of fields

      • getColumn

        public adams.core.base.BaseString[] getColumn()
        Returns the list of fields that identify a column.
        Returns:
        the list of fields

      • columnTipText

        public String columnTipText()
        Returns the tip text for this property.
        Returns:
        tip text for this property suitable for displaying in the GUI or for listing the options.

      • setSwapRowsAndColumns

        public void setSwapRowsAndColumns​(boolean value)
        Sets whether to swap rows and columns.
        Parameters:
        value - true if to swap rows and columns

      • getSwapRowsAndColumns

        public boolean getSwapRowsAndColumns()
        Returns whether to swap rows and columns.
        Returns:
        true if swapping rows and columns

      • swapRowsAndColumnsTipText

        public String swapRowsAndColumnsTipText()
        Returns the tip text for this property.
        Returns:
        tip text for this property suitable for displaying in the GUI or for listing the options.

      • setOutputFormat

        public void setOutputFormat​(weka.experiment.ResultMatrix value)
        Sets the output format to use for generating the output.
        Parameters:
        value - the format

      • getOutputFormat

        public weka.experiment.ResultMatrix getOutputFormat()
        Returns the output format in use for generating the output.
        Returns:
        the format

      • outputFormatTipText

        public String outputFormatTipText()
        Returns the tip text for this property.
        Returns:
        tip text for this property suitable for displaying in the GUI or for listing the options.

      • setOutputHeader

        public void setOutputHeader​(boolean value)
        Sets whether to output the header of the result matrix as well.
        Parameters:
        value - true if to output the header as well

      • getOutputHeader

        public boolean getOutputHeader()
        Returns whether to output the header of the result matrix as well.
        Returns:
        true if to output the header as well

      • outputHeaderTipText

        public String outputHeaderTipText()
        Returns the tip text for this property.
        Returns:
        tip text for this property suitable for displaying in the GUI or for listing the options.

      • accepts

        public Class[] accepts()
        Returns the class that the consumer accepts.
        Returns:
        weka.experiment.Experiment.class, weka.core.Instances.class

      • determineColumnNames

        protected List<String> determineColumnNames​(adams.core.base.BaseString[] list,
                                            String defaultList)
        Returns a vector with column names of the dataset, listed in "list". If a column cannot be found or the list is empty the ones from the default list are returned.
        Parameters:
        list - list of attribute names
        defaultList - the default list of attribute names
        Returns:
        a vector containing attribute names

      • getData

        protected weka.core.Instances getData​(weka.experiment.Experiment exp)
                               throws Exception
        Loads the experimental results.
        Parameters:
        exp - the experiment to evaluate
        Returns:
        the results
        Throws:
        Exception - If reading fails.

      • getTester

        protected weka.experiment.Tester getTester​(weka.core.Instances data)
                                    throws Exception
        Sets up the testing algorithm and returns it.
        Parameters:
        data - the experimental data
        Returns:
        the configured testing algorithm
        Throws:
        Exception - If something goes wrong, like testing algorithm of result matrix cannot be instantiated

      • evaluateExperiment

        protected void evaluateExperiment​(weka.core.Instances data)
                           throws Exception
        Evaluates the experiment data.
        Parameters:
        data - the data to evaluate
        Throws:
        Exception - If something goes wrong, like loading data fails or comparison field invalid

      • doExecute

        protected String doExecute()
        Executes the flow item.
        Specified by:
        doExecute in class adams.flow.core.AbstractActor
        Returns:
        null if everything is fine, otherwise error message

      • generates

        public Class[] generates()
        Returns the class of objects that it generates.
        Returns:
        java.lang.String.class