Optimize grid with X-validation and Performance Costs with wrong optimum

Steffen_wSteffen_w Member Posts: 3 Contributor I
edited November 2018 in Help

Hi!

I'm trying to optimize a k-nearest neighbour inside a x-validation. The performance is measured by the performance cost operator and the x-validation delivers an average value for the missclassification costs.

When i put the whole process inside the optimize grid operator, the performance seem not to have any impact on the selection of optimal parameters. By logging every run of the process, i can identify much better average cost results. Can anyone give me a hint on what i'm doing wrong?

 

Thanks in advance

Steffen

Best Answer

  • mschmitzmschmitz Posts: 2,316  RM Data Scientist
    Solution Accepted

    Dear  Steffen,

     

    you need to set the class ordering. The behaviour without that is odd. I would have expected an error and not that it does something wrong. If you set the class ordering in Performance (Costs) it works. See attached process.

     

    Another comment: The minimum found by log and by optimize are different. The reason is because one is logging the macro the other the micro performance (weighted and unweighted average of the k-folds). On a bigger data set, this should not make a difference.

     

    ~Martin

    Spoiler
    <?xml version="1.0" encoding="UTF-8" standalone="no"?>
    <process version="7.1.001">
    <context>
    <input/>
    <output/>
    <macros/>
    </context>
    <operator activated="true" class="process" compatibility="7.1.001" expanded="true" name="Process">
    <process expanded="true">
    <operator activated="true" class="retrieve" compatibility="7.1.001" expanded="true" height="68" name="Retrieve Golf" width="90" x="112" y="34">
    <parameter key="repository_entry" value="//Samples/data/Golf"/>
    </operator>
    <operator activated="true" class="optimize_parameters_grid" compatibility="7.1.001" expanded="true" height="103" name="Optimize Parameters (Grid)" width="90" x="313" y="34">
    <list key="parameters">
    <parameter key="k-NN.k" value="[1.0;100;99;linear]"/>
    </list>
    <parameter key="error_handling" value="ignore error"/>
    <process expanded="true">
    <operator activated="true" class="x_validation" compatibility="7.1.001" expanded="true" height="124" name="X-Validation" width="90" x="313" y="34">
    <parameter key="sampling_type" value="stratified sampling"/>
    <parameter key="use_local_random_seed" value="true"/>
    <process expanded="true">
    <operator activated="true" class="k_nn" compatibility="7.1.001" expanded="true" height="82" name="k-NN" width="90" x="253" y="34">
    <parameter key="k" value="100"/>
    </operator>
    <connect from_port="training" to_op="k-NN" to_port="training set"/>
    <connect from_op="k-NN" from_port="model" to_port="model"/>
    <portSpacing port="source_training" spacing="0"/>
    <portSpacing port="sink_model" spacing="0"/>
    <portSpacing port="sink_through 1" spacing="0"/>
    </process>
    <process expanded="true">
    <operator activated="true" class="apply_model" compatibility="7.1.001" expanded="true" height="82" name="Apply Model" width="90" x="45" y="34">
    <list key="application_parameters"/>
    </operator>
    <operator activated="true" class="performance_binominal_classification" compatibility="7.1.001" expanded="true" height="82" name="Performance" width="90" x="179" y="34"/>
    <operator activated="true" class="performance_costs" compatibility="7.1.001" expanded="true" height="82" name="Performance (2)" width="90" x="313" y="85">
    <parameter key="cost_matrix" value="[0.0 25.0;1.0 0.0]"/>
    <enumeration key="class_order_definition">
    <parameter key="class_name" value="yes"/>
    <parameter key="class_name" value="no"/>
    </enumeration>
    </operator>
    <connect from_port="model" to_op="Apply Model" to_port="model"/>
    <connect from_port="test set" to_op="Apply Model" to_port="unlabelled data"/>
    <connect from_op="Apply Model" from_port="labelled data" to_op="Performance" to_port="labelled data"/>
    <connect from_op="Performance" from_port="example set" to_op="Performance (2)" to_port="example set"/>
    <connect from_op="Performance (2)" from_port="performance" to_port="averagable 1"/>
    <portSpacing port="source_model" spacing="0"/>
    <portSpacing port="source_test set" spacing="0"/>
    <portSpacing port="source_through 1" spacing="0"/>
    <portSpacing port="sink_averagable 1" spacing="0"/>
    <portSpacing port="sink_averagable 2" spacing="0"/>
    </process>
    </operator>
    <operator activated="true" class="log" compatibility="7.1.001" expanded="true" height="82" name="Log" width="90" x="447" y="85">
    <list key="log">
    <parameter key="k" value="operator.k-NN.parameter.k"/>
    <parameter key="performance" value="operator.X-Validation.value.performance"/>
    </list>
    </operator>
    <connect from_port="input 1" to_op="X-Validation" to_port="training"/>
    <connect from_op="X-Validation" from_port="averagable 1" to_op="Log" to_port="through 1"/>
    <connect from_op="Log" from_port="through 1" to_port="performance"/>
    <portSpacing port="source_input 1" spacing="0"/>
    <portSpacing port="source_input 2" spacing="0"/>
    <portSpacing port="sink_performance" spacing="0"/>
    <portSpacing port="sink_result 1" spacing="0"/>
    </process>
    </operator>
    <connect from_op="Retrieve Golf" from_port="output" to_op="Optimize Parameters (Grid)" to_port="input 1"/>
    <connect from_op="Optimize Parameters (Grid)" from_port="performance" to_port="result 1"/>
    <connect from_op="Optimize Parameters (Grid)" from_port="parameter" to_port="result 2"/>
    <portSpacing port="source_input 1" spacing="0"/>
    <portSpacing port="sink_result 1" spacing="0"/>
    <portSpacing port="sink_result 2" spacing="0"/>
    <portSpacing port="sink_result 3" spacing="0"/>
    </process>
    </operator>
    </process>

     

     

     

    - Head of Data Science Services at RapidMiner -
    Dortmund, Germany

Answers

  • yyhuangyyhuang Administrator, Employee, RapidMiner Certified Analyst, RapidMiner Certified Expert, Member Posts: 271  RM Data Scientist

    Could you post your process here for us to investigate that? Thx

  • Steffen_wSteffen_w Member Posts: 3 Contributor I
    I created a test-process on the golf-dataset with the same situation. In the log, the optimum costs are at 0.350, the optimizer shows 10.2


    <?xml version="1.0" encoding="UTF-8" standalone="no"?>
    <process version="7.1.001">
    <context>
    <input/>
    <output/>
    <macros/>
    </context>
    <operator activated="true" class="process" compatibility="7.1.001" expanded="true" name="Process">
    <process expanded="true">
    <operator activated="true" class="retrieve" compatibility="7.1.001" expanded="true" height="68" name="Retrieve Golf" width="90" x="246" y="34">
    <parameter key="repository_entry" value="//Samples/data/Golf"/>
    </operator>
    <operator activated="true" class="optimize_parameters_grid" compatibility="7.1.001" expanded="true" height="103" name="Optimize Parameters (Grid)" width="90" x="581" y="34">
    <list key="parameters">
    <parameter key="k-NN.k" value="[1.0;100;99;linear]"/>
    </list>
    <parameter key="error_handling" value="ignore error"/>
    <process expanded="true">
    <operator activated="true" class="x_validation" compatibility="7.1.001" expanded="true" height="124" name="X-Validation" width="90" x="313" y="34">
    <parameter key="sampling_type" value="stratified sampling"/>
    <parameter key="use_local_random_seed" value="true"/>
    <process expanded="true">
    <operator activated="true" class="k_nn" compatibility="7.1.001" expanded="true" height="82" name="k-NN" width="90" x="253" y="34">
    <parameter key="k" value="100"/>
    </operator>
    <connect from_port="training" to_op="k-NN" to_port="training set"/>
    <connect from_op="k-NN" from_port="model" to_port="model"/>
    <portSpacing port="source_training" spacing="0"/>
    <portSpacing port="sink_model" spacing="0"/>
    <portSpacing port="sink_through 1" spacing="0"/>
    </process>
    <process expanded="true">
    <operator activated="true" class="apply_model" compatibility="7.1.001" expanded="true" height="82" name="Apply Model" width="90" x="45" y="34">
    <list key="application_parameters"/>
    </operator>
    <operator activated="true" class="performance_binominal_classification" compatibility="7.1.001" expanded="true" height="82" name="Performance" width="90" x="179" y="34"/>
    <operator activated="true" class="performance_costs" compatibility="7.1.001" expanded="true" height="82" name="Performance (2)" width="90" x="313" y="85">
    <parameter key="cost_matrix" value="[0.0 25.0;1.0 0.0]"/>
    <enumeration key="class_order_definition"/>
    </operator>
    <connect from_port="model" to_op="Apply Model" to_port="model"/>
    <connect from_port="test set" to_op="Apply Model" to_port="unlabelled data"/>
    <connect from_op="Apply Model" from_port="labelled data" to_op="Performance" to_port="labelled data"/>
    <connect from_op="Performance" from_port="example set" to_op="Performance (2)" to_port="example set"/>
    <connect from_op="Performance (2)" from_port="performance" to_port="averagable 1"/>
    <portSpacing port="source_model" spacing="0"/>
    <portSpacing port="source_test set" spacing="0"/>
    <portSpacing port="source_through 1" spacing="0"/>
    <portSpacing port="sink_averagable 1" spacing="0"/>
    <portSpacing port="sink_averagable 2" spacing="0"/>
    </process>
    </operator>
    <operator activated="true" class="log" compatibility="7.1.001" expanded="true" height="82" name="Log" width="90" x="447" y="85">
    <list key="log">
    <parameter key="k" value="operator.k-NN.parameter.k"/>
    <parameter key="performance" value="operator.X-Validation.value.performance"/>
    </list>
    </operator>
    <connect from_port="input 1" to_op="X-Validation" to_port="training"/>
    <connect from_op="X-Validation" from_port="averagable 1" to_op="Log" to_port="through 1"/>
    <connect from_op="Log" from_port="through 1" to_port="performance"/>
    <portSpacing port="source_input 1" spacing="0"/>
    <portSpacing port="source_input 2" spacing="0"/>
    <portSpacing port="sink_performance" spacing="0"/>
    <portSpacing port="sink_result 1" spacing="0"/>
    </process>
    </operator>
    <connect from_op="Retrieve Golf" from_port="output" to_op="Optimize Parameters (Grid)" to_port="input 1"/>
    <connect from_op="Optimize Parameters (Grid)" from_port="performance" to_port="result 1"/>
    <connect from_op="Optimize Parameters (Grid)" from_port="parameter" to_port="result 2"/>
    <portSpacing port="source_input 1" spacing="0"/>
    <portSpacing port="sink_result 1" spacing="0"/>
    <portSpacing port="sink_result 2" spacing="0"/>
    <portSpacing port="sink_result 3" spacing="0"/>
    </process>
    </operator>
    </process>
  • Steffen_wSteffen_w Member Posts: 3 Contributor I

    Dear Martin,

     

    thank you for your help! Now it's working!

     

    -Steffen

Sign In or Register to comment.